From ff453170329ad9f5414aa1e396130e82472f0b1b Mon Sep 17 00:00:00 2001 From: "Xiaojian \"JJ\" Deng" Date: Wed, 5 Jul 2023 20:40:43 -0400 Subject: [PATCH 01/54] Update models.py (#3020) Hopefully fixed error with "ValueError: Tokenizer class GPTNeoXTokenizer does not exist or is not currently imported." --- modules/models.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/modules/models.py b/modules/models.py index 160ca9e3..26a4880d 100644 --- a/modules/models.py +++ b/modules/models.py @@ -95,11 +95,18 @@ def load_tokenizer(model_name, model): if any(s in model_name.lower() for s in ['gpt-4chan', 'gpt4chan']) and Path(f"{shared.args.model_dir}/gpt-j-6B/").exists(): tokenizer = AutoTokenizer.from_pretrained(Path(f"{shared.args.model_dir}/gpt-j-6B/")) elif path_to_model.exists(): - tokenizer = AutoTokenizer.from_pretrained( - path_to_model, - trust_remote_code=shared.args.trust_remote_code, - use_fast=False - ) + try: + tokenizer = AutoTokenizer.from_pretrained( + path_to_model, + trust_remote_code=shared.args.trust_remote_code, + use_fast=False + ) + except ValueError: + tokenizer = AutoTokenizer.from_pretrained( + path_to_model, + trust_remote_code=shared.args.trust_remote_code, + use_fast=True + ) if tokenizer.__class__.__name__ == 'LlamaTokenizer': pairs = [ From 1f540fa4f8043f42b296280448cb51d2462466fb Mon Sep 17 00:00:00 2001 From: Fernando Tarin Morales Date: Fri, 7 Jul 2023 14:22:39 +0900 Subject: [PATCH 02/54] Added the format to be able to finetune Vicuna1.1 models (#3037) --- training/formats/vicuna-format.json | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 training/formats/vicuna-format.json diff --git a/training/formats/vicuna-format.json b/training/formats/vicuna-format.json new file mode 100644 index 00000000..c1aa4f15 --- /dev/null +++ b/training/formats/vicuna-format.json @@ -0,0 +1,3 @@ +{ + "instruction,output": "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.\n\nUSER: %instruction%\n\nASSISTANT: %output%" +} From d7e14e1f78865ba6d702fa5b567ae48beeef9680 Mon Sep 17 00:00:00 2001 From: Fernando Tarin Morales Date: Fri, 7 Jul 2023 14:24:07 +0900 Subject: [PATCH 03/54] Fixed the param name when loading a LoRA using a model loaded in 4 or 8 bits (#3036) --- modules/LoRA.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/modules/LoRA.py b/modules/LoRA.py index e54d5337..0eb56b56 100644 --- a/modules/LoRA.py +++ b/modules/LoRA.py @@ -114,11 +114,12 @@ def add_lora_transformers(lora_names): if len(lora_names) > 0: params = {} if not shared.args.cpu: - params['dtype'] = shared.model.dtype - if hasattr(shared.model, "hf_device_map"): - params['device_map'] = {"base_model.model." + k: v for k, v in shared.model.hf_device_map.items()} - elif shared.args.load_in_8bit: - params['device_map'] = {'': 0} + if shared.args.load_in_4bit or shared.args.load_in_8bit: + params['peft_type'] = shared.model.dtype + else: + params['dtype'] = shared.model.dtype + if hasattr(shared.model, "hf_device_map"): + params['device_map'] = {"base_model.model." + k: v for k, v in shared.model.hf_device_map.items()} logger.info("Applying the following LoRAs to {}: {}".format(shared.model_name, ', '.join(lora_names))) shared.model = PeftModel.from_pretrained(shared.model, Path(f"{shared.args.lora_dir}/{lora_names[0]}"), adapter_name=lora_names[0], **params) From 9aee1064a3b1fe4d10575961f4fa0d5403600433 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Thu, 6 Jul 2023 22:24:52 -0700 Subject: [PATCH 04/54] Block a cloudfare request --- modules/block_requests.py | 32 ++++++++++++++++++++++++++++++-- server.py | 11 ++++++----- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/modules/block_requests.py b/modules/block_requests.py index 4358a820..775a9b14 100644 --- a/modules/block_requests.py +++ b/modules/block_requests.py @@ -1,19 +1,47 @@ +import builtins +import io + import requests from modules.logging_colors import logger +original_open = open +original_get = requests.get + class RequestBlocker: def __enter__(self): - self.original_get = requests.get requests.get = my_get def __exit__(self, exc_type, exc_value, traceback): - requests.get = self.original_get + requests.get = original_get + + +class OpenMonkeyPatch: + + def __enter__(self): + builtins.open = my_open + + def __exit__(self, exc_type, exc_value, traceback): + builtins.open = original_open def my_get(url, **kwargs): logger.info('Unwanted HTTP request redirected to localhost :)') kwargs.setdefault('allow_redirects', True) return requests.api.request('get', 'http://127.0.0.1/', **kwargs) + + +# Kindly provided by our friend WizardLM-30B +def my_open(*args, **kwargs): + filename = str(args[0]) + if filename.endswith('index.html'): + with original_open(*args, **kwargs) as f: + file_contents = f.read() + + file_contents = file_contents.replace(b'', b'') + file_contents = file_contents.replace(b'cdnjs.cloudflare.com', b'127.0.0.1') + return io.BytesIO(file_contents) + else: + return original_open(*args, **kwargs) diff --git a/server.py b/server.py index 5c224e36..18474d3b 100644 --- a/server.py +++ b/server.py @@ -2,7 +2,7 @@ import os import warnings from modules.logging_colors import logger -from modules.block_requests import RequestBlocker +from modules.block_requests import OpenMonkeyPatch, RequestBlocker os.environ['GRADIO_ANALYTICS_ENABLED'] = 'False' os.environ['BITSANDBYTES_NOWELCOME'] = '1' @@ -1068,10 +1068,11 @@ def create_interface(): # Launch the interface shared.gradio['interface'].queue() - if shared.args.listen: - shared.gradio['interface'].launch(prevent_thread_lock=True, share=shared.args.share, server_name=shared.args.listen_host or '0.0.0.0', server_port=shared.args.listen_port, inbrowser=shared.args.auto_launch, auth=auth) - else: - shared.gradio['interface'].launch(prevent_thread_lock=True, share=shared.args.share, server_port=shared.args.listen_port, inbrowser=shared.args.auto_launch, auth=auth) + with OpenMonkeyPatch(): + if shared.args.listen: + shared.gradio['interface'].launch(prevent_thread_lock=True, share=shared.args.share, server_name=shared.args.listen_host or '0.0.0.0', server_port=shared.args.listen_port, inbrowser=shared.args.auto_launch, auth=auth) + else: + shared.gradio['interface'].launch(prevent_thread_lock=True, share=shared.args.share, server_port=shared.args.listen_port, inbrowser=shared.args.auto_launch, auth=auth) if __name__ == "__main__": From c21b73ff37d9d2613509dd1238aca15fd327510e Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 7 Jul 2023 09:09:14 -0700 Subject: [PATCH 05/54] Minor change to ui.py --- modules/ui.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/modules/ui.py b/modules/ui.py index f019bc87..35049533 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -67,7 +67,6 @@ def list_model_elements(): 'alpha_value' ] - for i in range(torch.cuda.device_count()): elements.append(f'gpu_memory_{i}') @@ -76,7 +75,6 @@ def list_model_elements(): def list_interface_input_elements(): elements = [ - 'preset_menu', 'max_new_tokens', 'seed', 'temperature', @@ -158,8 +156,7 @@ def apply_interface_values(state, use_persistent=False): if len(state) == 0: return [gr.update() for k in elements] # Dummy, do nothing else: - ans = [state[k] if k in state else gr.update() for k in elements] - return ans + return [state[k] if k in state else gr.update() for k in elements] class ToolButton(gr.Button, gr.components.FormComponent): From 1ba2e88551f968cd74478fd02218a62869336ac5 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 7 Jul 2023 09:09:23 -0700 Subject: [PATCH 06/54] Add truncation to exllama --- modules/exllama.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/exllama.py b/modules/exllama.py index 80607579..f685a445 100644 --- a/modules/exllama.py +++ b/modules/exllama.py @@ -1,10 +1,10 @@ -import sys from pathlib import Path from torch import version as torch_version from modules import shared from modules.logging_colors import logger +from modules.text_generation import get_max_prompt_length try: from exllama.generator import ExLlamaGenerator @@ -90,7 +90,11 @@ class ExllamaModel: self.generator.disallow_tokens(None) self.generator.end_beam_search() + + # Tokenizing the input ids = self.generator.tokenizer.encode(prompt) + ids = ids[:, -get_max_prompt_length(state):] + self.generator.gen_begin_reuse(ids) initial_len = self.generator.sequence[0].shape[0] has_leading_space = False From b6643e5039ae210dbc54ae6aa0f4dcf90b2269a8 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 7 Jul 2023 09:11:30 -0700 Subject: [PATCH 07/54] Add decode functions to llama.cpp/exllama --- modules/exllama.py | 3 +++ modules/llamacpp_model.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/modules/exllama.py b/modules/exllama.py index f685a445..ecfb10a4 100644 --- a/modules/exllama.py +++ b/modules/exllama.py @@ -120,3 +120,6 @@ class ExllamaModel: def encode(self, string, **kwargs): return self.tokenizer.encode(string) + + def decode(self, string, **kwargs): + return self.tokenizer.decode(string)[0] diff --git a/modules/llamacpp_model.py b/modules/llamacpp_model.py index 10a852db..4899ad99 100644 --- a/modules/llamacpp_model.py +++ b/modules/llamacpp_model.py @@ -65,6 +65,9 @@ class LlamaCppModel: return self.model.tokenize(string) + def decode(self, tokens): + return self.model.detokenize(tokens) + def generate(self, prompt, state, callback=None): prompt = prompt if type(prompt) is str else prompt.decode() completion_chunks = self.model.create_completion( From 79679b3cfdce749badb85ac198ba5014946429aa Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 7 Jul 2023 16:40:57 -0700 Subject: [PATCH 08/54] Pin fastapi version (for #3042) --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 692fbc55..fe042b6e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ accelerate==0.20.3 colorama datasets einops +fastapi==0.95.2 flexgen==0.1.7 gradio_client==0.2.5 gradio==3.33.1 From acf24ebb49d0077796508ab05630d8d5ae3df14d Mon Sep 17 00:00:00 2001 From: Brandon McClure Date: Fri, 7 Jul 2023 17:54:53 -0600 Subject: [PATCH 09/54] Whisper_stt params for model, language, and auto_submit (#3031) --- extensions/whisper_stt/readme.md | 15 +++++++++++++++ extensions/whisper_stt/script.py | 32 +++++++++++++++++++++++--------- 2 files changed, 38 insertions(+), 9 deletions(-) create mode 100644 extensions/whisper_stt/readme.md diff --git a/extensions/whisper_stt/readme.md b/extensions/whisper_stt/readme.md new file mode 100644 index 00000000..cd9abbf6 --- /dev/null +++ b/extensions/whisper_stt/readme.md @@ -0,0 +1,15 @@ +# whisper_stt + +Allows you to enter your inputs in chat mode using your microphone. + +## Settings + +To adjust your default settings, you can add the following to your settings.yaml file. + +``` +whisper_stt-whipser_language: chinese +whisper_stt-whipser_model: tiny +whisper_stt-auto_submit: False +``` + +See source documentation for [model names](https://github.com/openai/whisper#available-models-and-languages) and (languages)[https://github.com/openai/whisper/blob/main/whisper/tokenizer.py] you can use. \ No newline at end of file diff --git a/extensions/whisper_stt/script.py b/extensions/whisper_stt/script.py index 32226404..44a9ac81 100644 --- a/extensions/whisper_stt/script.py +++ b/extensions/whisper_stt/script.py @@ -8,8 +8,15 @@ input_hijack = { 'value': ["", ""] } +# parameters which can be customized in settings.json of webui +params = { + 'whipser_language': 'english', + 'whipser_model': 'small.en', + 'auto_submit': True +} -def do_stt(audio): + +def do_stt(audio,whipser_model,whipser_language): transcription = "" r = sr.Recognizer() @@ -17,7 +24,7 @@ def do_stt(audio): audio_data = sr.AudioData(sample_rate=audio[0], frame_data=audio[1], sample_width=4) try: - transcription = r.recognize_whisper(audio_data, language="english", model="base.en") + transcription = r.recognize_whisper(audio_data, language=whipser_language, model=whipser_model) except sr.UnknownValueError: print("Whisper could not understand audio") except sr.RequestError as e: @@ -26,11 +33,10 @@ def do_stt(audio): return transcription -def auto_transcribe(audio, auto_submit): +def auto_transcribe(audio, auto_submit,whipser_model,whipser_language): if audio is None: return "", "" - - transcription = do_stt(audio) + transcription = do_stt(audio,whipser_model,whipser_language) if auto_submit: input_hijack.update({"state": True, "value": [transcription, transcription]}) @@ -38,10 +44,18 @@ def auto_transcribe(audio, auto_submit): def ui(): - with gr.Row(): - audio = gr.Audio(source="microphone") - auto_submit = gr.Checkbox(label='Submit the transcribed audio automatically', value=True) + with gr.Accordion("Whisper STT", open=True): + with gr.Row(): + audio = gr.Audio(source="microphone") + with gr.Row(): + with gr.Accordion("Settings", open=False): + auto_submit = gr.Checkbox(label='Submit the transcribed audio automatically', value=params['auto_submit']) + whipser_model = gr.Dropdown(label='Whisper Model', value=params['whipser_model'],choices=["tiny.en","base.en", "small.en","medium.en","tiny","base","small","medium","large"]) + whipser_language = gr.Dropdown(label='Whisper Language', value=params['whipser_language'],choices=["chinese","german","spanish","russian","korean","french","japanese","portuguese","turkish","polish","catalan","dutch","arabic","swedish","italian","indonesian","hindi","finnish","vietnamese","hebrew","ukrainian","greek","malay","czech","romanian","danish","hungarian","tamil","norwegian","thai","urdu","croatian","bulgarian","lithuanian","latin","maori","malayalam","welsh","slovak","telugu","persian","latvian","bengali","serbian","azerbaijani","slovenian","kannada","estonian","macedonian","breton","basque","icelandic","armenian","nepali","mongolian","bosnian","kazakh","albanian","swahili","galician","marathi","punjabi","sinhala","khmer","shona","yoruba","somali","afrikaans","occitan","georgian","belarusian","tajik","sindhi","gujarati","amharic","yiddish","lao","uzbek","faroese","haitian creole","pashto","turkmen","nynorsk","maltese","sanskrit","luxembourgish","myanmar","tibetan","tagalog","malagasy","assamese","tatar","hawaiian","lingala","hausa","bashkir","javanese","sundanese"]) audio.change( - auto_transcribe, [audio, auto_submit], [shared.gradio['textbox'], audio]).then( + auto_transcribe, [audio, auto_submit,whipser_model,whipser_language], [shared.gradio['textbox'], audio]).then( None, auto_submit, None, _js="(check) => {if (check) { document.getElementById('Generate').click() }}") + whipser_model.change(lambda x: params.update({"whipser_model": x}), whipser_model, None) + whipser_language.change(lambda x: params.update({"whipser_language": x}), whipser_language, None) + auto_submit.change(lambda x: params.update({"auto_submit": x}), auto_submit, None) From 5ac4e4da8b6c4e6ad046248ae24c6633c0987a54 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Sat, 8 Jul 2023 10:22:54 -0700 Subject: [PATCH 10/54] Make --model work with argument like models/folder_name --- modules/models.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/models.py b/modules/models.py index 26a4880d..cd01899e 100644 --- a/modules/models.py +++ b/modules/models.py @@ -61,6 +61,10 @@ def load_model(model_name, loader=None): 'ExLlama_HF': ExLlama_HF_loader } + p = Path(model_name) + if p.exists(): + model_name = p.parts[-1] + if loader is None: if shared.args.loader is not None: loader = shared.args.loader From 70b088843d605f5693dcb0ebc904246e9e042e65 Mon Sep 17 00:00:00 2001 From: Chris Rude Date: Sat, 8 Jul 2023 19:21:20 -0700 Subject: [PATCH 11/54] fix for issue #2475: Streaming api deadlock (#3048) --- extensions/api/streaming_api.py | 130 +++++++++++++++++--------------- extensions/api/util.py | 42 +++++++++++ 2 files changed, 112 insertions(+), 60 deletions(-) diff --git a/extensions/api/streaming_api.py b/extensions/api/streaming_api.py index 2a7fb62b..88359e3e 100644 --- a/extensions/api/streaming_api.py +++ b/extensions/api/streaming_api.py @@ -4,7 +4,7 @@ from threading import Thread from websockets.server import serve -from extensions.api.util import build_parameters, try_start_cloudflared +from extensions.api.util import build_parameters, try_start_cloudflared, with_api_lock from modules import shared from modules.chat import generate_chat_reply from modules.text_generation import generate_reply @@ -12,72 +12,82 @@ from modules.text_generation import generate_reply PATH = '/api/v1/stream' +@with_api_lock +async def _handle_stream_message(websocket, message): + message = json.loads(message) + + prompt = message['prompt'] + generate_params = build_parameters(message) + stopping_strings = generate_params.pop('stopping_strings') + generate_params['stream'] = True + + generator = generate_reply( + prompt, generate_params, stopping_strings=stopping_strings, is_chat=False) + + # As we stream, only send the new bytes. + skip_index = 0 + message_num = 0 + + for a in generator: + to_send = a[skip_index:] + if to_send is None or chr(0xfffd) in to_send: # partial unicode character, don't send it yet. + continue + + await websocket.send(json.dumps({ + 'event': 'text_stream', + 'message_num': message_num, + 'text': to_send + })) + + await asyncio.sleep(0) + skip_index += len(to_send) + message_num += 1 + + await websocket.send(json.dumps({ + 'event': 'stream_end', + 'message_num': message_num + })) + + +@with_api_lock +async def _handle_chat_stream_message(websocket, message): + body = json.loads(message) + + user_input = body['user_input'] + generate_params = build_parameters(body, chat=True) + generate_params['stream'] = True + regenerate = body.get('regenerate', False) + _continue = body.get('_continue', False) + + generator = generate_chat_reply( + user_input, generate_params, regenerate=regenerate, _continue=_continue, loading_message=False) + + message_num = 0 + for a in generator: + await websocket.send(json.dumps({ + 'event': 'text_stream', + 'message_num': message_num, + 'history': a + })) + + await asyncio.sleep(0) + message_num += 1 + + await websocket.send(json.dumps({ + 'event': 'stream_end', + 'message_num': message_num + })) + + async def _handle_connection(websocket, path): if path == '/api/v1/stream': async for message in websocket: - message = json.loads(message) - - prompt = message['prompt'] - generate_params = build_parameters(message) - stopping_strings = generate_params.pop('stopping_strings') - generate_params['stream'] = True - - generator = generate_reply( - prompt, generate_params, stopping_strings=stopping_strings, is_chat=False) - - # As we stream, only send the new bytes. - skip_index = 0 - message_num = 0 - - for a in generator: - to_send = a[skip_index:] - if to_send is None or chr(0xfffd) in to_send: # partial unicode character, don't send it yet. - continue - - await websocket.send(json.dumps({ - 'event': 'text_stream', - 'message_num': message_num, - 'text': to_send - })) - - await asyncio.sleep(0) - skip_index += len(to_send) - message_num += 1 - - await websocket.send(json.dumps({ - 'event': 'stream_end', - 'message_num': message_num - })) + await _handle_stream_message(websocket, message) elif path == '/api/v1/chat-stream': async for message in websocket: - body = json.loads(message) - - user_input = body['user_input'] - generate_params = build_parameters(body, chat=True) - generate_params['stream'] = True - regenerate = body.get('regenerate', False) - _continue = body.get('_continue', False) - - generator = generate_chat_reply( - user_input, generate_params, regenerate=regenerate, _continue=_continue, loading_message=False) - - message_num = 0 - for a in generator: - await websocket.send(json.dumps({ - 'event': 'text_stream', - 'message_num': message_num, - 'history': a - })) - - await asyncio.sleep(0) - message_num += 1 - - await websocket.send(json.dumps({ - 'event': 'stream_end', - 'message_num': message_num - })) + await _handle_chat_stream_message(websocket, message) else: print(f'Streaming api: unknown path: {path}') diff --git a/extensions/api/util.py b/extensions/api/util.py index 6896c29f..a89365ce 100644 --- a/extensions/api/util.py +++ b/extensions/api/util.py @@ -1,3 +1,6 @@ +import asyncio +import functools +import threading import time import traceback from threading import Thread @@ -8,6 +11,13 @@ from modules.chat import load_character_memoized from modules.presets import load_preset_memoized +# We use a thread local to store the asyncio lock, so that each thread +# has its own lock. This isn't strictly necessary, but it makes it +# such that if we can support multiple worker threads in the future, +# thus handling multiple requests in parallel. +api_tls = threading.local() + + def build_parameters(body, chat=False): generate_params = { @@ -97,3 +107,35 @@ def _start_cloudflared(port: int, max_attempts: int = 3, on_start: Optional[Call time.sleep(3) raise Exception('Could not start cloudflared.') + + +def _get_api_lock(tls) -> asyncio.Lock: + """ + The streaming and blocking API implementations each run on their own + thread, and multiplex requests using asyncio. If multiple outstanding + requests are received at once, we will try to acquire the shared lock + shared.generation_lock multiple times in succession in the same thread, + which will cause a deadlock. + + To avoid this, we use this wrapper function to block on an asyncio + lock, and then try and grab the shared lock only while holding + the asyncio lock. + """ + if not hasattr(tls, "asyncio_lock"): + tls.asyncio_lock = asyncio.Lock() + + return tls.asyncio_lock + + +def with_api_lock(func): + """ + This decorator should be added to all streaming API methods which + require access to the shared.generation_lock. It ensures that the + tls.asyncio_lock is acquired before the method is called, and + released afterwards. + """ + @functools.wraps(func) + async def api_wrapper(*args, **kwargs): + async with _get_api_lock(api_tls): + return await func(*args, **kwargs) + return api_wrapper From 74ea7522a0576213f464a08f64ed87a93a1375a2 Mon Sep 17 00:00:00 2001 From: Forkoz <59298527+Ph0rk0z@users.noreply.github.com> Date: Sun, 9 Jul 2023 04:03:43 +0000 Subject: [PATCH 12/54] Lora fixes for AutoGPTQ (#2818) --- modules/LoRA.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/modules/LoRA.py b/modules/LoRA.py index 0eb56b56..2eade078 100644 --- a/modules/LoRA.py +++ b/modules/LoRA.py @@ -9,9 +9,9 @@ from modules.models import reload_model def add_lora_to_model(lora_names): - if 'GPTQForCausalLM' in shared.model.__class__.__name__: + if 'GPTQForCausalLM' in shared.model.__class__.__name__ or shared.args.loader == 'AutoGPTQ': add_lora_autogptq(lora_names) - elif shared.model.__class__.__name__ in ['ExllamaModel', 'ExllamaHF']: + elif shared.model.__class__.__name__ in ['ExllamaModel', 'ExllamaHF'] or shared.args.loader == 'ExLlama': add_lora_exllama(lora_names) else: add_lora_transformers(lora_names) @@ -66,15 +66,16 @@ def add_lora_autogptq(lora_names): logger.error("This version of AutoGPTQ does not support LoRA. You need to install from source or wait for a new release.") return - if len(lora_names) == 0: - if len(shared.lora_names) > 0: - reload_model() + if len(lora_names) == 0: + reload_model() shared.lora_names = [] return else: if len(lora_names) > 1: logger.warning('AutoGPTQ can only work with 1 LoRA at the moment. Only the first one in the list will be loaded.') + if not shared.args.no_inject_fused_attention: + logger.warning('Fused Atttention + AutoGPTQ may break Lora loading. Disable it.') peft_config = GPTQLoraConfig( inference_mode=True, From 463aac2d6530f889efd38df8df3ca9dab5377146 Mon Sep 17 00:00:00 2001 From: "Salvador E. Tropea" Date: Sun, 9 Jul 2023 01:08:20 -0300 Subject: [PATCH 13/54] [Added] google_translate activate param (#2961) - So you can quickly enable/disable it, otherwise you must select English to disable it, and then your language to enable it again. --- extensions/google_translate/script.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/extensions/google_translate/script.py b/extensions/google_translate/script.py index 63226107..5dfdbcd0 100644 --- a/extensions/google_translate/script.py +++ b/extensions/google_translate/script.py @@ -2,6 +2,7 @@ import gradio as gr from deep_translator import GoogleTranslator params = { + "activate": True, "language string": "ja", } @@ -13,6 +14,8 @@ def input_modifier(string): This function is applied to your text inputs before they are fed into the model. """ + if not params['activate']: + return string return GoogleTranslator(source=params['language string'], target='en').translate(string) @@ -21,6 +24,8 @@ def output_modifier(string): """ This function is applied to the model outputs. """ + if not params['activate']: + return string return GoogleTranslator(source='en', target=params['language string']).translate(string) @@ -40,7 +45,12 @@ def ui(): language_name = list(language_codes.keys())[list(language_codes.values()).index(params['language string'])] # Gradio elements - language = gr.Dropdown(value=language_name, choices=[k for k in language_codes], label='Language') + with gr.Row(): + activate = gr.Checkbox(value=params['activate'], label='Activate translation') + + with gr.Row(): + language = gr.Dropdown(value=language_name, choices=[k for k in language_codes], label='Language') # Event functions to update the parameters in the backend + activate.change(lambda x: params.update({"activate": x}), activate, None) language.change(lambda x: params.update({"language string": language_codes[x]}), language, None) From 161d984e801d755cd942b9707ba866ebfa6a2853 Mon Sep 17 00:00:00 2001 From: ofirkris Date: Sun, 9 Jul 2023 23:22:24 +0300 Subject: [PATCH 14/54] Bump llama-cpp-python version (#3072) Bump llama-cpp-python version to 0.1.69 --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index fe042b6e..e5530242 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,8 +20,8 @@ transformers==4.30.2 git+https://github.com/huggingface/peft@03eb378eb914fbee709ff7c86ba5b1d033b89524 bitsandbytes==0.39.1; platform_system != "Windows" https://github.com/jllllll/bitsandbytes-windows-webui/releases/download/wheels/bitsandbytes-0.39.1-py3-none-win_amd64.whl; platform_system == "Windows" -llama-cpp-python==0.1.68; platform_system != "Windows" -https://github.com/abetlen/llama-cpp-python/releases/download/v0.1.68/llama_cpp_python-0.1.68-cp310-cp310-win_amd64.whl; platform_system == "Windows" +llama-cpp-python==0.1.69; platform_system != "Windows" +https://github.com/abetlen/llama-cpp-python/releases/download/v0.1.69/llama_cpp_python-0.1.69-cp310-cp310-win_amd64.whl; platform_system == "Windows" https://github.com/PanQiWei/AutoGPTQ/releases/download/v0.2.2/auto_gptq-0.2.2+cu117-cp310-cp310-win_amd64.whl; platform_system == "Windows" https://github.com/PanQiWei/AutoGPTQ/releases/download/v0.2.2/auto_gptq-0.2.2+cu117-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" https://github.com/jllllll/exllama/releases/download/0.0.5/exllama-0.0.5+cu117-cp310-cp310-win_amd64.whl; platform_system == "Windows" From c7058afb402bd381d1983837b779c106217120b3 Mon Sep 17 00:00:00 2001 From: tianchen zhong Date: Sun, 9 Jul 2023 13:22:56 -0700 Subject: [PATCH 15/54] Add new possible bin file name regex (#3070) --- download-model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/download-model.py b/download-model.py index 2642c405..dedd5f6c 100644 --- a/download-model.py +++ b/download-model.py @@ -73,7 +73,7 @@ class ModelDownloader: if not is_lora and fname.endswith(('adapter_config.json', 'adapter_model.bin')): is_lora = True - is_pytorch = re.match("(pytorch|adapter)_model.*\.bin", fname) + is_pytorch = re.match("(pytorch|adapter|gptq)_model.*\.bin", fname) is_safetensors = re.match(".*\.safetensors", fname) is_pt = re.match(".*\.pt", fname) is_ggml = re.match(".*ggml.*\.bin", fname) From f8dbd7519bdda69ce4577d74865ed7e5d20749fc Mon Sep 17 00:00:00 2001 From: jllllll <3887729+jllllll@users.noreply.github.com> Date: Mon, 10 Jul 2023 17:35:59 -0500 Subject: [PATCH 16/54] Bump exllama module version (#3087) https://github.com/turboderp/exllama/compare/d769533b6fe938d5533ffd3673e4751e5afa5a09...e61d4d31d44bd4dd3380fc773509c870ba74cb9f --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index e5530242..229ba169 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,5 +24,5 @@ llama-cpp-python==0.1.69; platform_system != "Windows" https://github.com/abetlen/llama-cpp-python/releases/download/v0.1.69/llama_cpp_python-0.1.69-cp310-cp310-win_amd64.whl; platform_system == "Windows" https://github.com/PanQiWei/AutoGPTQ/releases/download/v0.2.2/auto_gptq-0.2.2+cu117-cp310-cp310-win_amd64.whl; platform_system == "Windows" https://github.com/PanQiWei/AutoGPTQ/releases/download/v0.2.2/auto_gptq-0.2.2+cu117-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" -https://github.com/jllllll/exllama/releases/download/0.0.5/exllama-0.0.5+cu117-cp310-cp310-win_amd64.whl; platform_system == "Windows" -https://github.com/jllllll/exllama/releases/download/0.0.5/exllama-0.0.5+cu117-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" +https://github.com/jllllll/exllama/releases/download/0.0.6/exllama-0.0.6+cu117-cp310-cp310-win_amd64.whl; platform_system == "Windows" +https://github.com/jllllll/exllama/releases/download/0.0.6/exllama-0.0.6+cu117-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" From a81cdd1367c3a4cf77e37fc0ab21ba7990bba843 Mon Sep 17 00:00:00 2001 From: ofirkris Date: Tue, 11 Jul 2023 01:36:15 +0300 Subject: [PATCH 17/54] Bump cpp llama version (#3081) Bump cpp llama version to 0.1.70 --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 229ba169..77c6cebb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,8 +20,8 @@ transformers==4.30.2 git+https://github.com/huggingface/peft@03eb378eb914fbee709ff7c86ba5b1d033b89524 bitsandbytes==0.39.1; platform_system != "Windows" https://github.com/jllllll/bitsandbytes-windows-webui/releases/download/wheels/bitsandbytes-0.39.1-py3-none-win_amd64.whl; platform_system == "Windows" -llama-cpp-python==0.1.69; platform_system != "Windows" -https://github.com/abetlen/llama-cpp-python/releases/download/v0.1.69/llama_cpp_python-0.1.69-cp310-cp310-win_amd64.whl; platform_system == "Windows" +llama-cpp-python==0.1.70; platform_system != "Windows" +https://github.com/abetlen/llama-cpp-python/releases/download/v0.1.70/llama_cpp_python-0.1.70-cp310-cp310-win_amd64.whl; platform_system == "Windows" https://github.com/PanQiWei/AutoGPTQ/releases/download/v0.2.2/auto_gptq-0.2.2+cu117-cp310-cp310-win_amd64.whl; platform_system == "Windows" https://github.com/PanQiWei/AutoGPTQ/releases/download/v0.2.2/auto_gptq-0.2.2+cu117-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" https://github.com/jllllll/exllama/releases/download/0.0.6/exllama-0.0.6+cu117-cp310-cp310-win_amd64.whl; platform_system == "Windows" From f4aa11cef67d10cbd1ce3050bcbd624db6b9c7f7 Mon Sep 17 00:00:00 2001 From: Josh XT <102809327+Josh-XT@users.noreply.github.com> Date: Tue, 11 Jul 2023 17:38:26 -0400 Subject: [PATCH 18/54] Add default environment variable values to docker compose file (#3102) Add default environment variable values to docker compose file --- docker/docker-compose.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index bc59dc3b..46b27580 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -5,13 +5,13 @@ services: context: . args: # specify which cuda version your card supports: https://developer.nvidia.com/cuda-gpus - TORCH_CUDA_ARCH_LIST: ${TORCH_CUDA_ARCH_LIST} - WEBUI_VERSION: ${WEBUI_VERSION} + TORCH_CUDA_ARCH_LIST: ${TORCH_CUDA_ARCH_LIST:-7.5} + WEBUI_VERSION: ${WEBUI_VERSION:-HEAD} env_file: .env ports: - - "${HOST_PORT}:${CONTAINER_PORT}" - - "${HOST_API_PORT}:${CONTAINER_API_PORT}" - - "${HOST_API_STREAM_PORT}:${CONTAINER_API_STREAM_PORT}" + - "${HOST_PORT:-7860}:${CONTAINER_PORT:-7860}" + - "${HOST_API_PORT:-5000}:${CONTAINER_API_PORT:-5000}" + - "${HOST_API_STREAM_PORT:-5005}:${CONTAINER_API_STREAM_PORT:-5005}" stdin_open: true tty: true volumes: From 987d522b559a535a43ed8489893f78be53b302df Mon Sep 17 00:00:00 2001 From: Vadim Peretokin Date: Tue, 11 Jul 2023 23:40:55 +0200 Subject: [PATCH 19/54] Fix API example for loading models (#3101) --- api-examples/api-example-model.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api-examples/api-example-model.py b/api-examples/api-example-model.py index 8e1e3002..1e108a2d 100644 --- a/api-examples/api-example-model.py +++ b/api-examples/api-example-model.py @@ -54,7 +54,7 @@ def complex_model_load(model): 'action': 'load', 'model_name': model, 'args': { - 'gptq_for_llama': False, # Use AutoGPTQ by default, set to True for gptq-for-llama + 'loader': 'AutoGPTQ', 'bf16': False, 'load_in_8bit': False, @@ -74,7 +74,7 @@ def complex_model_load(model): 'rwkv_strategy': None, 'rwkv_cuda_on': False, - # b&b 4-bit + # b&b 4-bit #'load_in_4bit': False, #'compute_dtype': 'float16', #'quant_type': 'nf4', @@ -148,11 +148,11 @@ if __name__ == '__main__': except Exception as e: print (f"❌ {model} FAIL Exception: {repr(e)}") - + # 0,1,1,2,3,5,8,13, is the fibonacci sequence, the next number is 21. # Some results below. -""" $ ./model-api-example.py +""" $ ./model-api-example.py Model: 4bit_gpt4-x-alpaca-13b-native-4bit-128g-cuda Lora(s): [] truncation_length = 2048 From fdd596f98fb42b775214d6bfc3dd3142a0f10c43 Mon Sep 17 00:00:00 2001 From: jllllll <3887729+jllllll@users.noreply.github.com> Date: Tue, 11 Jul 2023 16:41:24 -0500 Subject: [PATCH 20/54] Bump bitsandbytes Windows wheel (#3097) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 77c6cebb..93d07060 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ scipy transformers==4.30.2 git+https://github.com/huggingface/peft@03eb378eb914fbee709ff7c86ba5b1d033b89524 bitsandbytes==0.39.1; platform_system != "Windows" -https://github.com/jllllll/bitsandbytes-windows-webui/releases/download/wheels/bitsandbytes-0.39.1-py3-none-win_amd64.whl; platform_system == "Windows" +https://github.com/jllllll/bitsandbytes-windows-webui/releases/download/wheels/bitsandbytes-0.40.0-py3-none-win_amd64.whl; platform_system == "Windows" llama-cpp-python==0.1.70; platform_system != "Windows" https://github.com/abetlen/llama-cpp-python/releases/download/v0.1.70/llama_cpp_python-0.1.70-cp310-cp310-win_amd64.whl; platform_system == "Windows" https://github.com/PanQiWei/AutoGPTQ/releases/download/v0.2.2/auto_gptq-0.2.2+cu117-cp310-cp310-win_amd64.whl; platform_system == "Windows" From 61102899cd3f0de493b243051a5da3947aa322eb Mon Sep 17 00:00:00 2001 From: FartyPants Date: Tue, 11 Jul 2023 17:46:59 -0400 Subject: [PATCH 21/54] google flan T5 download fix (#3080) --- download-model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/download-model.py b/download-model.py index dedd5f6c..d6b2ebff 100644 --- a/download-model.py +++ b/download-model.py @@ -77,7 +77,7 @@ class ModelDownloader: is_safetensors = re.match(".*\.safetensors", fname) is_pt = re.match(".*\.pt", fname) is_ggml = re.match(".*ggml.*\.bin", fname) - is_tokenizer = re.match("(tokenizer|ice).*\.model", fname) + is_tokenizer = re.match("(tokenizer|ice|spiece).*\.model", fname) is_text = re.match(".*\.(txt|json|py|md)", fname) or is_tokenizer if any((is_pytorch, is_safetensors, is_pt, is_ggml, is_tokenizer, is_text)): if 'lfs' in dict[i]: From 8db7e857b1690ea38733ce0fde940aafb0318aea Mon Sep 17 00:00:00 2001 From: Ahmad Fahadh Ilyas <37577369+fahadh4ilyas@users.noreply.github.com> Date: Wed, 12 Jul 2023 04:48:08 +0700 Subject: [PATCH 22/54] Add token authorization for downloading model (#3067) --- download-model.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/download-model.py b/download-model.py index d6b2ebff..34986c75 100644 --- a/download-model.py +++ b/download-model.py @@ -30,6 +30,8 @@ class ModelDownloader: self.s.mount('https://huggingface.co', HTTPAdapter(max_retries=max_retries)) if os.getenv('HF_USER') is not None and os.getenv('HF_PASS') is not None: self.s.auth = (os.getenv('HF_USER'), os.getenv('HF_PASS')) + if os.getenv('HF_TOKEN') is not None: + self.s.headers = {'authorization': f'Bearer {os.getenv("HF_TOKEN")}'} def sanitize_model_and_branch_names(self, model, branch): if model[-1] == '/': From 3e7feb699c12b4dde6ed9f3781efbfc699c3b5d1 Mon Sep 17 00:00:00 2001 From: matatonic <73265741+matatonic@users.noreply.github.com> Date: Tue, 11 Jul 2023 17:50:08 -0400 Subject: [PATCH 23/54] extensions/openai: Major openai extension updates & fixes (#3049) * many openai updates * total reorg & cleanup. * fixups * missing import os for images * +moderations, custom_stopping_strings, more fixes * fix bugs in completion streaming * moderation fix (flagged) * updated moderation categories --------- Co-authored-by: Matthew Ashton --- extensions/openai/README.md | 3 +- extensions/openai/completions.py | 599 +++++++++++++++++++ extensions/openai/defaults.py | 64 ++ extensions/openai/edits.py | 102 ++++ extensions/openai/embeddings.py | 50 ++ extensions/openai/errors.py | 27 + extensions/openai/images.py | 48 ++ extensions/openai/models.py | 77 +++ extensions/openai/moderations.py | 70 +++ extensions/openai/requirements.txt | 3 +- extensions/openai/script.py | 907 +++++------------------------ extensions/openai/tokens.py | 37 ++ extensions/openai/utils.py | 26 + 13 files changed, 1246 insertions(+), 767 deletions(-) create mode 100644 extensions/openai/completions.py create mode 100644 extensions/openai/defaults.py create mode 100644 extensions/openai/edits.py create mode 100644 extensions/openai/embeddings.py create mode 100644 extensions/openai/errors.py create mode 100644 extensions/openai/images.py create mode 100644 extensions/openai/models.py create mode 100644 extensions/openai/moderations.py create mode 100644 extensions/openai/tokens.py create mode 100644 extensions/openai/utils.py diff --git a/extensions/openai/README.md b/extensions/openai/README.md index 0f775bbf..7bbc1e83 100644 --- a/extensions/openai/README.md +++ b/extensions/openai/README.md @@ -218,12 +218,11 @@ but there are some exceptions. | ✅❌ | langchain | https://github.com/hwchase17/langchain | OPENAI_API_BASE=http://127.0.0.1:5001/v1 even with a good 30B-4bit model the result is poor so far. It assumes zero shot python/json coding. Some model tailored prompt formatting improves results greatly. | | ✅❌ | Auto-GPT | https://github.com/Significant-Gravitas/Auto-GPT | OPENAI_API_BASE=http://127.0.0.1:5001/v1 Same issues as langchain. Also assumes a 4k+ context | | ✅❌ | babyagi | https://github.com/yoheinakajima/babyagi | OPENAI_API_BASE=http://127.0.0.1:5001/v1 | +| ❌ | guidance | https://github.com/microsoft/guidance | logit_bias and logprobs not yet supported | ## Future plans -* better error handling * model changing, esp. something for swapping loras or embedding models * consider switching to FastAPI + starlette for SSE (openai SSE seems non-standard) -* do something about rate limiting or locking requests for completions, most systems will only be able handle a single request at a time before OOM ## Bugs? Feedback? Comments? Pull requests? diff --git a/extensions/openai/completions.py b/extensions/openai/completions.py new file mode 100644 index 00000000..ab1879b7 --- /dev/null +++ b/extensions/openai/completions.py @@ -0,0 +1,599 @@ +import time +import yaml +import tiktoken +import torch +import torch.nn.functional as F + +from transformers import LogitsProcessor, LogitsProcessorList + +from modules import shared +from modules.text_generation import encode, decode, generate_reply + +from extensions.openai.defaults import get_default_req_params, default, clamp +from extensions.openai.utils import end_line, debug_msg +from extensions.openai.errors import * + + +# Thanks to @Cypherfox [Cypherfoxy] for the logits code, blame to @matatonic +class LogitsBiasProcessor(LogitsProcessor): + def __init__(self, logit_bias={}): + self.logit_bias = logit_bias + super().__init__() + + def __call__(self, input_ids: torch.LongTensor, logits: torch.FloatTensor) -> torch.FloatTensor: + if self.logit_bias: + keys = list([int(key) for key in self.logit_bias.keys()]) + values = list([int(val) for val in self.logit_bias.values()]) + logits[0, keys] += torch.tensor(values).cuda() + + return logits + + +class LogprobProcessor(LogitsProcessor): + def __init__(self, logprobs=None): + self.logprobs = logprobs + self.token_alternatives = {} + super().__init__() + + def __call__(self, input_ids: torch.LongTensor, logits: torch.FloatTensor) -> torch.FloatTensor: + if self.logprobs is not None: # 0-5 + log_e_probabilities = F.log_softmax(logits, dim=1) + # XXX hack. should find the selected token and include the prob of that + # ... but we just +1 here instead because we don't know it yet. + top_values, top_indices = torch.topk(log_e_probabilities, k=self.logprobs + 1) + top_tokens = [ decode(tok) for tok in top_indices[0] ] + self.token_alternatives = dict(zip(top_tokens, top_values[0].tolist())) + return logits + + +def convert_logprobs_to_tiktoken(model, logprobs): + try: + encoder = tiktoken.encoding_for_model(model) + # just pick the first one if it encodes to multiple tokens... 99.9% not required and maybe worse overall. + return dict([ (encoder.decode([encoder.encode(token)[0]]), prob) for token, prob in logprobs.items() ]) + except KeyError: + # assume native tokens if we can't find the tokenizer + return logprobs + + +def marshal_common_params(body): + # Request Parameters + # Try to use openai defaults or map them to something with the same intent + + req_params = get_default_req_params() + + # Common request parameters + req_params['truncation_length'] = shared.settings['truncation_length'] + req_params['add_bos_token'] = shared.settings.get('add_bos_token', req_params['add_bos_token']) + req_params['seed'] = shared.settings.get('seed', req_params['seed']) + req_params['custom_stopping_strings'] = shared.settings['custom_stopping_strings'] + + # OpenAI API Parameters + # model - ignored for now, TODO: When we can reliably load a model or lora from a name only change this + req_params['requested_model'] = body.get('model', shared.model_name) + + req_params['suffix'] = default(body, 'suffix', req_params['suffix']) + req_params['temperature'] = clamp(default(body, 'temperature', req_params['temperature']), 0.001, 1.999) # fixup absolute 0.0/2.0 + req_params['top_p'] = clamp(default(body, 'top_p', req_params['top_p']), 0.001, 1.0) + n = default(body, 'n', 1) + if n != 1: + raise InvalidRequestError(message="Only n = 1 is supported.", param='n') + + if 'stop' in body: # str or array, max len 4 (ignored) + if isinstance(body['stop'], str): + req_params['stopping_strings'] = [body['stop']] # non-standard parameter + elif isinstance(body['stop'], list): + req_params['stopping_strings'] = body['stop'] + + # presence_penalty - ignored + # frequency_penalty - ignored + # user - ignored + + logits_processor = [] + logit_bias = body.get('logit_bias', None) + if logit_bias: # {str: float, ...} + # XXX convert tokens from tiktoken based on requested model + # Ex.: 'logit_bias': {'1129': 100, '11442': 100, '16243': 100} + try: + encoder = tiktoken.encoding_for_model(req_params['requested_model']) + new_logit_bias = {} + for logit, bias in logit_bias.items(): + for x in encode(encoder.decode([int(logit)]))[0]: + new_logit_bias[str(int(x))] = bias + print(logit_bias, '->', new_logit_bias) + logit_bias = new_logit_bias + except KeyError: + pass # assume native tokens if we can't find the tokenizer + + logits_processor = [LogitsBiasProcessor(logit_bias)] + + logprobs = None # coming to chat eventually + if 'logprobs' in body: + logprobs = default(body, 'logprobs', 0) # maybe cap at topk? don't clamp 0-5. + req_params['logprob_proc'] = LogprobProcessor(logprobs) + logits_processor.extend([req_params['logprob_proc']]) + else: + logprobs = None + + if logits_processor: # requires logits_processor support + req_params['logits_processor'] = LogitsProcessorList(logits_processor) + + return req_params + + +def messages_to_prompt(body: dict, req_params: dict, max_tokens): + # functions + if body.get('functions', []): # chat only + raise InvalidRequestError(message="functions is not supported.", param='functions') + if body.get('function_call', ''): # chat only, 'none', 'auto', {'name': 'func'} + raise InvalidRequestError(message="function_call is not supported.", param='function_call') + + if not 'messages' in body: + raise InvalidRequestError(message="messages is required", param='messages') + + messages = body['messages'] + + role_formats = { + 'user': 'user: {message}\n', + 'assistant': 'assistant: {message}\n', + 'system': '{message}', + 'context': 'You are a helpful assistant. Answer as concisely as possible.', + 'prompt': 'assistant:', + } + + if not 'stopping_strings' in req_params: + req_params['stopping_strings'] = [] + + # Instruct models can be much better + if shared.settings['instruction_template']: + try: + instruct = yaml.safe_load(open(f"characters/instruction-following/{shared.settings['instruction_template']}.yaml", 'r')) + + template = instruct['turn_template'] + system_message_template = "{message}" + system_message_default = instruct['context'] + bot_start = template.find('<|bot|>') # So far, 100% of instruction templates have this token + user_message_template = template[:bot_start].replace('<|user-message|>', '{message}').replace('<|user|>', instruct['user']) + bot_message_template = template[bot_start:].replace('<|bot-message|>', '{message}').replace('<|bot|>', instruct['bot']) + bot_prompt = bot_message_template[:bot_message_template.find('{message}')].rstrip(' ') + + role_formats = { + 'user': user_message_template, + 'assistant': bot_message_template, + 'system': system_message_template, + 'context': system_message_default, + 'prompt': bot_prompt, + } + + if 'Alpaca' in shared.settings['instruction_template']: + req_params['stopping_strings'].extend(['\n###']) + elif instruct['user']: # WizardLM and some others have no user prompt. + req_params['stopping_strings'].extend(['\n' + instruct['user'], instruct['user']]) + + debug_msg(f"Loaded instruction role format: {shared.settings['instruction_template']}") + + except Exception as e: + req_params['stopping_strings'].extend(['\nuser:']) + + print(f"Exception: When loading characters/instruction-following/{shared.settings['instruction_template']}.yaml: {repr(e)}") + print("Warning: Loaded default instruction-following template for model.") + + else: + req_params['stopping_strings'].extend(['\nuser:']) + print("Warning: Loaded default instruction-following template for model.") + + system_msgs = [] + chat_msgs = [] + + # You are ChatGPT, a large language model trained by OpenAI. Answer as concisely as possible. Knowledge cutoff: {knowledge_cutoff} Current date: {current_date} + context_msg = role_formats['system'].format(message=role_formats['context']) if role_formats['context'] else '' + context_msg = end_line(context_msg) + + # Maybe they sent both? This is not documented in the API, but some clients seem to do this. + if 'prompt' in body: + context_msg = end_line(role_formats['system'].format(message=body['prompt'])) + context_msg + + for m in messages: + role = m['role'] + content = m['content'] + # name = m.get('name', None) + # function_call = m.get('function_call', None) # user name or function name with output in content + msg = role_formats[role].format(message=content) + if role == 'system': + system_msgs.extend([msg]) + elif role == 'function': + raise InvalidRequestError(message="role: function is not supported.", param='messages') + else: + chat_msgs.extend([msg]) + + system_msg = '\n'.join(system_msgs) + system_msg = end_line(system_msg) + + prompt = system_msg + context_msg + ''.join(chat_msgs) + role_formats['prompt'] + + token_count = len(encode(prompt)[0]) + + if token_count >= req_params['truncation_length']: + err_msg = f"This model maximum context length is {req_params['truncation_length']} tokens. However, your messages resulted in over {token_count} tokens." + raise InvalidRequestError(message=err_msg) + + if max_tokens > 0 and token_count + max_tokens > req_params['truncation_length']: + err_msg = f"This model maximum context length is {req_params['truncation_length']} tokens. However, your messages resulted in over {token_count} tokens and max_tokens is {max_tokens}." + print(f"Warning: ${err_msg}") + #raise InvalidRequestError(message=err_msg) + + return prompt, token_count + + +def chat_completions(body: dict, is_legacy: bool=False) -> dict: + # Chat Completions + object_type = 'chat.completions' + created_time = int(time.time()) + cmpl_id = "chatcmpl-%d" % (int(time.time()*1000000000)) + resp_list = 'data' if is_legacy else 'choices' + + # common params + req_params = marshal_common_params(body) + req_params['stream'] = False + requested_model = req_params.pop('requested_model') + logprob_proc = req_params.pop('logprob_proc', None) + req_params['top_k'] = 20 # There is no best_of/top_k param for chat, but it is much improved with a higher top_k. + + # chat default max_tokens is 'inf', but also flexible + max_tokens = 0 + max_tokens_str = 'length' if is_legacy else 'max_tokens' + if max_tokens_str in body: + max_tokens = default(body, max_tokens_str, req_params['truncation_length']) + req_params['max_new_tokens'] = max_tokens + else: + req_params['max_new_tokens'] = req_params['truncation_length'] + + # format the prompt from messages + prompt, token_count = messages_to_prompt(body, req_params, max_tokens) + + # generate reply ####################################### + debug_msg({'prompt': prompt, 'req_params': req_params}) + stopping_strings = req_params.pop('stopping_strings', []) + logprob_proc = req_params.pop('logprob_proc', None) + generator = generate_reply(prompt, req_params, stopping_strings=stopping_strings, is_chat=False) + + answer = '' + for a in generator: + answer = a + + # strip extra leading space off new generated content + if answer and answer[0] == ' ': + answer = answer[1:] + + completion_token_count = len(encode(answer)[0]) + stop_reason = "stop" + if token_count + completion_token_count >= req_params['truncation_length'] or completion_token_count >= max_tokens: + stop_reason = "length" + + resp = { + "id": cmpl_id, + "object": object_type, + "created": created_time, + "model": shared.model_name, # TODO: add Lora info? + resp_list: [{ + "index": 0, + "finish_reason": stop_reason, + "message": {"role": "assistant", "content": answer} + }], + "usage": { + "prompt_tokens": token_count, + "completion_tokens": completion_token_count, + "total_tokens": token_count + completion_token_count + } + } + if logprob_proc: # not official for chat yet + top_logprobs = convert_logprobs_to_tiktoken(model=requested_model, logprobs=logprob_proc.token_alternatives) + resp[resp_list][0]["logprobs"] = {'top_logprobs': [top_logprobs]} + # else: + # resp[resp_list][0]["logprobs"] = None + + return resp + + +# generator +def stream_chat_completions(body: dict, is_legacy: bool=False): + + # Chat Completions + stream_object_type = 'chat.completions.chunk' + created_time = int(time.time()) + cmpl_id = "chatcmpl-%d" % (int(time.time()*1000000000)) + resp_list = 'data' if is_legacy else 'choices' + + # common params + req_params = marshal_common_params(body) + req_params['stream'] = True + requested_model = req_params.pop('requested_model') + logprob_proc = req_params.pop('logprob_proc', None) + req_params['top_k'] = 20 # There is no best_of/top_k param for chat, but it is much improved with a higher top_k. + + # chat default max_tokens is 'inf', but also flexible + max_tokens = 0 + max_tokens_str = 'length' if is_legacy else 'max_tokens' + if max_tokens_str in body: + max_tokens = default(body, max_tokens_str, req_params['truncation_length']) + req_params['max_new_tokens'] = max_tokens + else: + req_params['max_new_tokens'] = req_params['truncation_length'] + + # format the prompt from messages + prompt, token_count = messages_to_prompt(body, req_params, max_tokens) + + def chat_streaming_chunk(content): + # begin streaming + chunk = { + "id": cmpl_id, + "object": stream_object_type, + "created": created_time, + "model": shared.model_name, + resp_list: [{ + "index": 0, + "finish_reason": None, + # So yeah... do both methods? delta and messages. + "message": {'role': 'assistant', 'content': content}, + "delta": {'role': 'assistant', 'content': content}, + }], + } + + if logprob_proc: # not official for chat yet + top_logprobs = convert_logprobs_to_tiktoken(model=requested_model, logprobs=logprob_proc.token_alternatives) + chunk[resp_list][0]["logprobs"] = {'top_logprobs': [top_logprobs]} + #else: + # chunk[resp_list][0]["logprobs"] = None + return chunk + + yield chat_streaming_chunk('') + + # generate reply ####################################### + debug_msg({'prompt': prompt, 'req_params': req_params}) + + stopping_strings = req_params.pop('stopping_strings', []) + logprob_proc = req_params.pop('logprob_proc', None) + + generator = generate_reply(prompt, req_params, stopping_strings=stopping_strings, is_chat=False) + + answer = '' + seen_content = '' + completion_token_count = 0 + + for a in generator: + answer = a + + len_seen = len(seen_content) + new_content = answer[len_seen:] + + if not new_content or chr(0xfffd) in new_content: # partial unicode character, don't send it yet. + continue + + seen_content = answer + + # strip extra leading space off new generated content + if len_seen == 0 and new_content[0] == ' ': + new_content = new_content[1:] + + completion_token_count += len(encode(new_content)[0]) + chunk = chat_streaming_chunk(new_content) + + yield chunk + + + stop_reason = "stop" + if token_count + completion_token_count >= req_params['truncation_length'] or completion_token_count >= max_tokens: + stop_reason = "length" + + chunk = chat_streaming_chunk('') + chunk[resp_list][0]['finish_reason'] = stop_reason + chunk['usage'] = { + "prompt_tokens": token_count, + "completion_tokens": completion_token_count, + "total_tokens": token_count + completion_token_count + } + + yield chunk + + +def completions(body: dict, is_legacy: bool=False): + # Legacy + # Text Completions + object_type = 'text_completion' + created_time = int(time.time()) + cmpl_id = "conv-%d" % (int(time.time()*1000000000)) + resp_list = 'data' if is_legacy else 'choices' + + # ... encoded as a string, array of strings, array of tokens, or array of token arrays. + prompt_str = 'context' if is_legacy else 'prompt' + if not prompt_str in body: + raise InvalidRequestError("Missing required input", param=prompt_str) + + prompt = body[prompt_str] + if isinstance(prompt, list): + if prompt and isinstance(prompt[0], int): + try: + encoder = tiktoken.encoding_for_model(requested_model) + prompt = encode(encoder.decode(prompt))[0] + except KeyError: + prompt = decode(prompt)[0] + else: + raise InvalidRequestError(message="API Batched generation not yet supported.", param=prompt_str) + + # common params + req_params = marshal_common_params(body) + req_params['stream'] = False + max_tokens_str = 'length' if is_legacy else 'max_tokens' + max_tokens = default(body, max_tokens_str, req_params['max_new_tokens']) + req_params['max_new_tokens'] = max_tokens + requested_model = req_params.pop('requested_model') + logprob_proc = req_params.pop('logprob_proc', None) + + token_count = len(encode(prompt)[0]) + + if token_count + max_tokens > req_params['truncation_length']: + err_msg = f"The token count of your prompt ({token_count}) plus max_tokens ({max_tokens}) cannot exceed the model's context length ({req_params['truncation_length']})." + #print(f"Warning: ${err_msg}") + raise InvalidRequestError(message=err_msg, param=max_tokens_str) + + req_params['echo'] = default(body, 'echo', req_params['echo']) + req_params['top_k'] = default(body, 'best_of', req_params['top_k']) + + # generate reply ####################################### + debug_msg({'prompt': prompt, 'req_params': req_params}) + stopping_strings = req_params.pop('stopping_strings', []) + logprob_proc = req_params.pop('logprob_proc', None) + generator = generate_reply(prompt, req_params, stopping_strings=stopping_strings, is_chat=False) + + answer = '' + + for a in generator: + answer = a + + # strip extra leading space off new generated content + if answer and answer[0] == ' ': + answer = answer[1:] + + completion_token_count = len(encode(answer)[0]) + stop_reason = "stop" + if token_count + completion_token_count >= req_params['truncation_length'] or completion_token_count >= max_tokens: + stop_reason = "length" + + resp = { + "id": cmpl_id, + "object": object_type, + "created": created_time, + "model": shared.model_name, # TODO: add Lora info? + resp_list: [{ + "index": 0, + "finish_reason": stop_reason, + "text": answer, + }], + "usage": { + "prompt_tokens": token_count, + "completion_tokens": completion_token_count, + "total_tokens": token_count + completion_token_count + } + } + + if logprob_proc: + top_logprobs = convert_logprobs_to_tiktoken(model=requested_model, logprobs=logprob_proc.token_alternatives) + resp[resp_list][0]["logprobs"] = {'top_logprobs': [top_logprobs]} + else: + resp[resp_list][0]["logprobs"] = None + + return resp + + +# generator +def stream_completions(body: dict, is_legacy: bool=False): + # Legacy + # Text Completions + #object_type = 'text_completion' + stream_object_type = 'text_completion.chunk' + created_time = int(time.time()) + cmpl_id = "conv-%d" % (int(time.time()*1000000000)) + resp_list = 'data' if is_legacy else 'choices' + + # ... encoded as a string, array of strings, array of tokens, or array of token arrays. + prompt_str = 'context' if is_legacy else 'prompt' + if not prompt_str in body: + raise InvalidRequestError("Missing required input", param=prompt_str) + + prompt = body[prompt_str] + if isinstance(prompt, list): + if prompt and isinstance(prompt[0], int): + try: + encoder = tiktoken.encoding_for_model(requested_model) + prompt = encode(encoder.decode(prompt))[0] + except KeyError: + prompt = decode(prompt)[0] + else: + raise InvalidRequestError(message="API Batched generation not yet supported.", param=prompt_str) + + # common params + req_params = marshal_common_params(body) + req_params['stream'] = True + max_tokens_str = 'length' if is_legacy else 'max_tokens' + max_tokens = default(body, max_tokens_str, req_params['max_new_tokens']) + req_params['max_new_tokens'] = max_tokens + requested_model = req_params.pop('requested_model') + logprob_proc = req_params.pop('logprob_proc', None) + + token_count = len(encode(prompt)[0]) + + if token_count + max_tokens > req_params['truncation_length']: + err_msg = f"The token count of your prompt ({token_count}) plus max_tokens ({max_tokens}) cannot exceed the model's context length ({req_params['truncation_length']})." + #print(f"Warning: ${err_msg}") + raise InvalidRequestError(message=err_msg, param=max_tokens_str) + + req_params['echo'] = default(body, 'echo', req_params['echo']) + req_params['top_k'] = default(body, 'best_of', req_params['top_k']) + + def text_streaming_chunk(content): + # begin streaming + chunk = { + "id": cmpl_id, + "object": stream_object_type, + "created": created_time, + "model": shared.model_name, + resp_list: [{ + "index": 0, + "finish_reason": None, + "text": content, + }], + } + if logprob_proc: + top_logprobs = convert_logprobs_to_tiktoken(model=requested_model, logprobs=logprob_proc.token_alternatives) + chunk[resp_list][0]["logprobs"] = {'top_logprobs': [top_logprobs]} + else: + chunk[resp_list][0]["logprobs"] = None + + return chunk + + yield text_streaming_chunk('') + + # generate reply ####################################### + debug_msg({'prompt': prompt, 'req_params': req_params}) + stopping_strings = req_params.pop('stopping_strings', []) + logprob_proc = req_params.pop('logprob_proc', None) + generator = generate_reply(prompt, req_params, stopping_strings=stopping_strings, is_chat=False) + + answer = '' + seen_content = '' + completion_token_count = 0 + + for a in generator: + answer = a + + len_seen = len(seen_content) + new_content = answer[len_seen:] + + if not new_content or chr(0xfffd) in new_content: # partial unicode character, don't send it yet. + continue + + seen_content = answer + + # strip extra leading space off new generated content + if len_seen == 0 and new_content[0] == ' ': + new_content = new_content[1:] + + chunk = text_streaming_chunk(new_content) + + completion_token_count += len(encode(new_content)[0]) + yield chunk + + + stop_reason = "stop" + if token_count + completion_token_count >= req_params['truncation_length'] or completion_token_count >= max_tokens: + stop_reason = "length" + + chunk = text_streaming_chunk('') + chunk[resp_list][0]["finish_reason"] = stop_reason + chunk["usage"] = { + "prompt_tokens": token_count, + "completion_tokens": completion_token_count, + "total_tokens": token_count + completion_token_count + } + + yield chunk diff --git a/extensions/openai/defaults.py b/extensions/openai/defaults.py new file mode 100644 index 00000000..822a6e24 --- /dev/null +++ b/extensions/openai/defaults.py @@ -0,0 +1,64 @@ +import copy + +# Slightly different defaults for OpenAI's API +# Data type is important, Ex. use 0.0 for a float 0 +default_req_params = { + 'max_new_tokens': 16, # 'Inf' for chat + 'temperature': 1.0, + 'top_p': 1.0, + 'top_k': 1, # choose 20 for chat in absence of another default + 'repetition_penalty': 1.18, + 'repetition_penalty_range': 0, + 'encoder_repetition_penalty': 1.0, + 'suffix': None, + 'stream': False, + 'echo': False, + 'seed': -1, + # 'n' : default(body, 'n', 1), # 'n' doesn't have a direct map + 'truncation_length': 2048, # first use shared.settings value + 'add_bos_token': True, + 'do_sample': True, + 'typical_p': 1.0, + 'epsilon_cutoff': 0.0, # In units of 1e-4 + 'eta_cutoff': 0.0, # In units of 1e-4 + 'tfs': 1.0, + 'top_a': 0.0, + 'min_length': 0, + 'no_repeat_ngram_size': 0, + 'num_beams': 1, + 'penalty_alpha': 0.0, + 'length_penalty': 1.0, + 'early_stopping': False, + 'mirostat_mode': 0, + 'mirostat_tau': 5.0, + 'mirostat_eta': 0.1, + 'ban_eos_token': False, + 'skip_special_tokens': True, + 'custom_stopping_strings': '', + # 'logits_processor' - conditionally passed + # 'stopping_strings' - temporarily used + # 'logprobs' - temporarily used + # 'requested_model' - temporarily used +} + +def get_default_req_params(): + return copy.deepcopy(default_req_params) + +# little helper to get defaults if arg is present but None and should be the same type as default. +def default(dic, key, default): + val = dic.get(key, default) + if type(val) != type(default): + # maybe it's just something like 1 instead of 1.0 + try: + v = type(default)(val) + if type(val)(v) == val: # if it's the same value passed in, it's ok. + return v + except: + pass + + val = default + return val + +def clamp(value, minvalue, maxvalue): + return max(minvalue, min(value, maxvalue)) + diff --git a/extensions/openai/edits.py b/extensions/openai/edits.py new file mode 100644 index 00000000..3c51dc68 --- /dev/null +++ b/extensions/openai/edits.py @@ -0,0 +1,102 @@ +import time +import yaml +import os +from modules import shared +from extensions.openai.defaults import get_default_req_params +from extensions.openai.utils import debug_msg +from extensions.openai.errors import * +from modules.text_generation import encode, generate_reply + + +def edits(instruction: str, input: str, temperature = 1.0, top_p = 1.0) -> dict: + + created_time = int(time.time()*1000) + + # Request parameters + req_params = get_default_req_params() + stopping_strings = [] + + # Alpaca is verbose so a good default prompt + default_template = ( + "Below is an instruction that describes a task, paired with an input that provides further context. " + "Write a response that appropriately completes the request.\n\n" + "### Instruction:\n{instruction}\n\n### Input:\n{input}\n\n### Response:\n" + ) + + instruction_template = default_template + + # Use the special instruction/input/response template for anything trained like Alpaca + if shared.settings['instruction_template']: + if 'Alpaca' in shared.settings['instruction_template']: + stopping_strings.extend(['\n###']) + else: + try: + instruct = yaml.safe_load(open(f"characters/instruction-following/{shared.settings['instruction_template']}.yaml", 'r')) + + template = instruct['turn_template'] + template = template\ + .replace('<|user|>', instruct.get('user', ''))\ + .replace('<|bot|>', instruct.get('bot', ''))\ + .replace('<|user-message|>', '{instruction}\n{input}') + + instruction_template = instruct.get('context', '') + template[:template.find('<|bot-message|>')].rstrip(' ') + if instruct['user']: + stopping_strings.extend(['\n' + instruct['user'], instruct['user'] ]) + + except Exception as e: + instruction_template = default_template + print(f"Exception: When loading characters/instruction-following/{shared.settings['instruction_template']}.yaml: {repr(e)}") + print("Warning: Loaded default instruction-following template (Alpaca) for model.") + else: + stopping_strings.extend(['\n###']) + print("Warning: Loaded default instruction-following template (Alpaca) for model.") + + edit_task = instruction_template.format(instruction=instruction, input=input) + + truncation_length = shared.settings['truncation_length'] + + token_count = len(encode(edit_task)[0]) + max_tokens = truncation_length - token_count + + if max_tokens < 1: + err_msg = f"This model maximum context length is {truncation_length} tokens. However, your messages resulted in over {truncation_length - max_tokens} tokens." + raise InvalidRequestError(err_msg, param='input') + + req_params['max_new_tokens'] = max_tokens + req_params['truncation_length'] = truncation_length + req_params['temperature'] = temperature + req_params['top_p'] = top_p + req_params['seed'] = shared.settings.get('seed', req_params['seed']) + req_params['add_bos_token'] = shared.settings.get('add_bos_token', req_params['add_bos_token']) + req_params['custom_stopping_strings'] = shared.settings['custom_stopping_strings'] + + debug_msg({'edit_template': edit_task, 'req_params': req_params, 'token_count': token_count}) + + generator = generate_reply(edit_task, req_params, stopping_strings=stopping_strings, is_chat=False) + + longest_stop_len = max([len(x) for x in stopping_strings] + [0]) + answer = '' + for a in generator: + answer = a + + # some reply's have an extra leading space to fit the instruction template, just clip it off from the reply. + if edit_task[-1] != '\n' and answer and answer[0] == ' ': + answer = answer[1:] + + completion_token_count = len(encode(answer)[0]) + + resp = { + "object": "edit", + "created": created_time, + "choices": [{ + "text": answer, + "index": 0, + }], + "usage": { + "prompt_tokens": token_count, + "completion_tokens": completion_token_count, + "total_tokens": token_count + completion_token_count + } + } + + return resp diff --git a/extensions/openai/embeddings.py b/extensions/openai/embeddings.py new file mode 100644 index 00000000..54e70ae7 --- /dev/null +++ b/extensions/openai/embeddings.py @@ -0,0 +1,50 @@ +import os +from sentence_transformers import SentenceTransformer +from extensions.openai.utils import float_list_to_base64, debug_msg +from extensions.openai.errors import * + +st_model = os.environ["OPENEDAI_EMBEDDING_MODEL"] if "OPENEDAI_EMBEDDING_MODEL" in os.environ else "all-mpnet-base-v2" +embeddings_model = None + +def load_embedding_model(model): + try: + emb_model = SentenceTransformer(model) + print(f"\nLoaded embedding model: {model}, max sequence length: {emb_model.max_seq_length}") + except Exception as e: + print(f"\nError: Failed to load embedding model: {model}") + raise ServiceUnavailableError(f"Error: Failed to load embedding model: {model}", internal_message = repr(e)) + + return emb_model + +def get_embeddings_model(): + global embeddings_model, st_model + if st_model and not embeddings_model: + embeddings_model = load_embedding_model(st_model) # lazy load the model + return embeddings_model + +def get_embeddings_model_name(): + global st_model + return st_model + +def embeddings(input: list, encoding_format: str): + + embeddings = get_embeddings_model().encode(input).tolist() + + if encoding_format == "base64": + data = [{"object": "embedding", "embedding": float_list_to_base64(emb), "index": n} for n, emb in enumerate(embeddings)] + else: + data = [{"object": "embedding", "embedding": emb, "index": n} for n, emb in enumerate(embeddings)] + + response = { + "object": "list", + "data": data, + "model": st_model, # return the real model + "usage": { + "prompt_tokens": 0, + "total_tokens": 0, + } + } + + debug_msg(f"Embeddings return size: {len(embeddings[0])}, number: {len(embeddings)}") + + return response \ No newline at end of file diff --git a/extensions/openai/errors.py b/extensions/openai/errors.py new file mode 100644 index 00000000..df66a300 --- /dev/null +++ b/extensions/openai/errors.py @@ -0,0 +1,27 @@ +class OpenAIError(Exception): + def __init__(self, message = None, code = 500, internal_message = ''): + self.message = message + self.code = code + self.internal_message = internal_message + def __repr__(self): + return "%s(message=%r, code=%d)" % ( + self.__class__.__name__, + self.message, + self.code, + ) + +class InvalidRequestError(OpenAIError): + def __init__(self, message, param, code = 400, error_type ='InvalidRequestError', internal_message = ''): + super(OpenAIError, self).__init__(message, code, error_type, internal_message) + self.param = param + def __repr__(self): + return "%s(message=%r, code=%d, param=%s)" % ( + self.__class__.__name__, + self.message, + self.code, + self.param, + ) + +class ServiceUnavailableError(OpenAIError): + def __init__(self, message = None, code = 500, error_type ='ServiceUnavailableError', internal_message = ''): + super(OpenAIError, self).__init__(message, code, error_type, internal_message) diff --git a/extensions/openai/images.py b/extensions/openai/images.py new file mode 100644 index 00000000..d911ed63 --- /dev/null +++ b/extensions/openai/images.py @@ -0,0 +1,48 @@ +import os +import time +import requests +from extensions.openai.errors import * + +def generations(prompt: str, size: str, response_format: str, n: int): + # Stable Diffusion callout wrapper for txt2img + # Low effort implementation for compatibility. With only "prompt" being passed and assuming DALL-E + # the results will be limited and likely poor. SD has hundreds of models and dozens of settings. + # If you want high quality tailored results you should just use the Stable Diffusion API directly. + # it's too general an API to try and shape the result with specific tags like "masterpiece", etc, + # Will probably work best with the stock SD models. + # SD configuration is beyond the scope of this API. + # At this point I will not add the edits and variations endpoints (ie. img2img) because they + # require changing the form data handling to accept multipart form data, also to properly support + # url return types will require file management and a web serving files... Perhaps later! + + width, height = [ int(x) for x in size.split('x') ] # ignore the restrictions on size + + # to hack on better generation, edit default payload. + payload = { + 'prompt': prompt, # ignore prompt limit of 1000 characters + 'width': width, + 'height': height, + 'batch_size': n, + 'restore_faces': True, # slightly less horrible + } + + resp = { + 'created': int(time.time()), + 'data': [] + } + + # TODO: support SD_WEBUI_AUTH username:password pair. + sd_url = f"{os.environ['SD_WEBUI_URL']}/sdapi/v1/txt2img" + + response = requests.post(url=sd_url, json=payload) + r = response.json() + if response.status_code != 200 or 'images' not in r: + raise ServiceUnavailableError(r.get('detail', [{'msg': 'Unknown error calling Stable Diffusion'}])[0]['msg'], code = response.status_code) + # r['parameters']... + for b64_json in r['images']: + if response_format == 'b64_json': + resp['data'].extend([{'b64_json': b64_json}]) + else: + resp['data'].extend([{'url': f'data:image/png;base64,{b64_json}'}]) # yeah it's lazy. requests.get() will not work with this + + return resp \ No newline at end of file diff --git a/extensions/openai/models.py b/extensions/openai/models.py new file mode 100644 index 00000000..bed5ed49 --- /dev/null +++ b/extensions/openai/models.py @@ -0,0 +1,77 @@ +from modules import shared +from modules.utils import get_available_models +from modules.models import load_model, unload_model +from modules.models_settings import (get_model_settings_from_yamls, + update_model_parameters) + +from extensions.openai.embeddings import get_embeddings_model_name +from extensions.openai.errors import * + +def get_current_model_list() -> list: + return [ shared.model_name ] # The real chat/completions model, maybe "None" + +def get_pseudo_model_list() -> list: + return [ # these are expected by so much, so include some here as a dummy + 'gpt-3.5-turbo', + 'text-embedding-ada-002', + ] + +def load_model(model_name: str) -> dict: + resp = { + "id": model_name, + "object": "engine", + "owner": "self", + "ready": True, + } + if model_name not in get_pseudo_model_list() + [ get_embeddings_model_name() ] + get_current_model_list(): # Real model only + # No args. Maybe it works anyways! + # TODO: hack some heuristics into args for better results + + shared.model_name = model_name + unload_model() + + model_settings = get_model_settings_from_yamls(shared.model_name) + shared.settings.update(model_settings) + update_model_parameters(model_settings, initial=True) + + if shared.settings['mode'] != 'instruct': + shared.settings['instruction_template'] = None + + shared.model, shared.tokenizer = load_model(shared.model_name) + + if not shared.model: # load failed. + shared.model_name = "None" + raise OpenAIError(f"Model load failed for: {shared.model_name}") + + return resp + + +def list_models(is_legacy: bool = False) -> dict: + # TODO: Lora's? + all_model_list = get_current_model_list() + [ get_embeddings_model_name() ] + get_pseudo_model_list() + get_available_models() + + models = {} + + if is_legacy: + models = [{ "id": id, "object": "engine", "owner": "user", "ready": True } for id in all_model_list ] + if not shared.model: + models[0]['ready'] = False + else: + models = [{ "id": id, "object": "model", "owned_by": "user", "permission": [] } for id in all_model_list ] + + resp = { + "object": "list", + "data": models, + } + + return resp + + +def model_info(model_name: str) -> dict: + return { + "id": model_name, + "object": "model", + "owned_by": "user", + "permission": [] + } + diff --git a/extensions/openai/moderations.py b/extensions/openai/moderations.py new file mode 100644 index 00000000..7daf0176 --- /dev/null +++ b/extensions/openai/moderations.py @@ -0,0 +1,70 @@ +import time +import numpy as np +from numpy.linalg import norm +from extensions.openai.embeddings import get_embeddings_model + + +moderations_disabled = False # return 0/false +category_embeddings = None +antonym_embeddings = None +categories = [ "sexual", "hate", "harassment", "self-harm", "sexual/minors", "hate/threatening", "violence/graphic", "self-harm/intent", "self-harm/instructions", "harassment/threatening", "violence" ] +flag_threshold = 0.5 + + +def get_category_embeddings(): + global category_embeddings, categories + if category_embeddings is None: + embeddings = get_embeddings_model().encode(categories).tolist() + category_embeddings = dict(zip(categories, embeddings)) + + return category_embeddings + + +def cosine_similarity(a, b): + return np.dot(a, b) / (norm(a) * norm(b)) + + +# seems most openai like with all-mpnet-base-v2 +def mod_score(a, b): + return 2.0 * np.dot(a, b) + + +def moderations(input): + global category_embeddings, categories, flag_threshold, moderations_disabled + results = { + "id": f"modr-{int(time.time()*1e9)}", + "model": "text-moderation-001", + "results": [], + } + + embeddings_model = get_embeddings_model() + if not embeddings_model or moderations_disabled: + results['results'] = [{ + 'categories': dict([ (C, False) for C in categories]), + 'category_scores': dict([ (C, 0.0) for C in categories]), + 'flagged': False, + }] + return results + + category_embeddings = get_category_embeddings() + + + # input, string or array + if isinstance(input, str): + input = [input] + + for in_str in input: + for ine in embeddings_model.encode([in_str]).tolist(): + category_scores = dict([ (C, mod_score(category_embeddings[C], ine)) for C in categories ]) + category_flags = dict([ (C, bool(category_scores[C] > flag_threshold)) for C in categories ]) + flagged = any(category_flags.values()) + + results['results'].extend([{ + 'flagged': flagged, + 'categories': category_flags, + 'category_scores': category_scores, + }]) + + print(results) + + return results \ No newline at end of file diff --git a/extensions/openai/requirements.txt b/extensions/openai/requirements.txt index 5193a0ac..56d567b8 100644 --- a/extensions/openai/requirements.txt +++ b/extensions/openai/requirements.txt @@ -1,2 +1,3 @@ flask_cloudflared==0.0.12 -sentence-transformers \ No newline at end of file +sentence-transformers +tiktoken \ No newline at end of file diff --git a/extensions/openai/script.py b/extensions/openai/script.py index 323d6823..1ff1eb6b 100644 --- a/extensions/openai/script.py +++ b/extensions/openai/script.py @@ -1,108 +1,27 @@ -import base64 import json import os -import time -import requests -import yaml -import numpy as np +import traceback from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from threading import Thread -from modules.utils import get_available_models -from modules.models import load_model, unload_model -from modules.models_settings import (get_model_settings_from_yamls, - update_model_parameters) from modules import shared -from modules.text_generation import encode, generate_reply + +from extensions.openai.tokens import token_count, token_encode, token_decode +import extensions.openai.models as OAImodels +import extensions.openai.edits as OAIedits +import extensions.openai.embeddings as OAIembeddings +import extensions.openai.images as OAIimages +import extensions.openai.moderations as OAImoderations +import extensions.openai.completions as OAIcompletions +from extensions.openai.errors import * +from extensions.openai.utils import debug_msg +from extensions.openai.defaults import (get_default_req_params, default, clamp) + params = { 'port': int(os.environ.get('OPENEDAI_PORT')) if 'OPENEDAI_PORT' in os.environ else 5001, } -debug = True if 'OPENEDAI_DEBUG' in os.environ else False - -# Slightly different defaults for OpenAI's API -# Data type is important, Ex. use 0.0 for a float 0 -default_req_params = { - 'max_new_tokens': 200, - 'temperature': 1.0, - 'top_p': 1.0, - 'top_k': 1, - 'repetition_penalty': 1.18, - 'repetition_penalty_range': 0, - 'encoder_repetition_penalty': 1.0, - 'suffix': None, - 'stream': False, - 'echo': False, - 'seed': -1, - # 'n' : default(body, 'n', 1), # 'n' doesn't have a direct map - 'truncation_length': 2048, - 'add_bos_token': True, - 'do_sample': True, - 'typical_p': 1.0, - 'epsilon_cutoff': 0.0, # In units of 1e-4 - 'eta_cutoff': 0.0, # In units of 1e-4 - 'tfs': 1.0, - 'top_a': 0.0, - 'min_length': 0, - 'no_repeat_ngram_size': 0, - 'num_beams': 1, - 'penalty_alpha': 0.0, - 'length_penalty': 1.0, - 'early_stopping': False, - 'mirostat_mode': 0, - 'mirostat_tau': 5.0, - 'mirostat_eta': 0.1, - 'ban_eos_token': False, - 'skip_special_tokens': True, - 'custom_stopping_strings': '', -} - -# Optional, install the module and download the model to enable -# v1/embeddings -try: - from sentence_transformers import SentenceTransformer -except ImportError: - pass - -st_model = os.environ["OPENEDAI_EMBEDDING_MODEL"] if "OPENEDAI_EMBEDDING_MODEL" in os.environ else "all-mpnet-base-v2" -embedding_model = None - -# little helper to get defaults if arg is present but None and should be the same type as default. -def default(dic, key, default): - val = dic.get(key, default) - if type(val) != type(default): - # maybe it's just something like 1 instead of 1.0 - try: - v = type(default)(val) - if type(val)(v) == val: # if it's the same value passed in, it's ok. - return v - except: - pass - - val = default - return val - - -def clamp(value, minvalue, maxvalue): - return max(minvalue, min(value, maxvalue)) - - -def float_list_to_base64(float_list): - # Convert the list to a float32 array that the OpenAPI client expects - float_array = np.array(float_list, dtype="float32") - - # Get raw bytes - bytes_array = float_array.tobytes() - - # Encode bytes into base64 - encoded_bytes = base64.b64encode(bytes_array) - - # Turn raw base64 encoded bytes into ASCII - ascii_string = encoded_bytes.decode('ascii') - return ascii_string - - class Handler(BaseHTTPRequestHandler): def send_access_control_headers(self): self.send_header("Access-Control-Allow-Origin", "*") @@ -118,11 +37,43 @@ class Handler(BaseHTTPRequestHandler): "Authorization" ) - def openai_error(self, message, code = 500, error_type = 'APIError', param = '', internal_message = ''): + def do_OPTIONS(self): + self.send_response(200) + self.send_access_control_headers() + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write("OK".encode('utf-8')) + + def start_sse(self): + self.send_response(200) + self.send_access_control_headers() + self.send_header('Content-Type', 'text/event-stream') + self.send_header('Cache-Control', 'no-cache') + # self.send_header('Connection', 'keep-alive') + self.end_headers() + + def send_sse(self, chunk: dict): + response = 'data: ' + json.dumps(chunk) + '\r\n\r\n' + debug_msg(response) + self.wfile.write(response.encode('utf-8')) + + def end_sse(self): + self.wfile.write('data: [DONE]\r\n\r\n'.encode('utf-8')) + + def return_json(self, ret: dict, code: int = 200, no_debug=False): self.send_response(code) self.send_access_control_headers() self.send_header('Content-Type', 'application/json') self.end_headers() + + response = json.dumps(ret) + r_utf8 = response.encode('utf-8') + self.wfile.write(r_utf8) + if not no_debug: + debug_msg(r_utf8) + + def openai_error(self, message, code = 500, error_type = 'APIError', param = '', internal_message = ''): + error_resp = { 'error': { 'message': message, @@ -132,121 +83,61 @@ class Handler(BaseHTTPRequestHandler): } } if internal_message: - error_resp['internal_message'] = internal_message + print(internal_message) + #error_resp['internal_message'] = internal_message - response = json.dumps(error_resp) - self.wfile.write(response.encode('utf-8')) + self.return_json(error_resp, code) + + def openai_error_handler(func): + def wrapper(self): + try: + func(self) + except ServiceUnavailableError as e: + self.openai_error(e.message, e.code, e.error_type, internal_message=e.internal_message) + except InvalidRequestError as e: + self.openai_error(e.message, e.code, e.error_type, e.param, internal_message=e.internal_message) + except OpenAIError as e: + self.openai_error(e.message, e.code, e.error_type, internal_message=e.internal_message) + except Exception as e: + self.openai_error(repr(e), 500, 'OpenAIError', internal_message=traceback.format_exc()) - def do_OPTIONS(self): - self.send_response(200) - self.send_access_control_headers() - self.send_header('Content-Type', 'application/json') - self.end_headers() - self.wfile.write("OK".encode('utf-8')) + return wrapper + @openai_error_handler def do_GET(self): - if self.path.startswith('/v1/engines') or self.path.startswith('/v1/models'): - current_model_list = [ shared.model_name ] # The real chat/completions model, maybe "None" - embeddings_model_list = [ st_model ] if embedding_model else [] # The real sentence transformer embeddings model - pseudo_model_list = [ # these are expected by so much, so include some here as a dummy - 'gpt-3.5-turbo', # /v1/chat/completions - 'text-curie-001', # /v1/completions, 2k context - 'text-davinci-002' # /v1/embeddings text-embedding-ada-002:1536, text-davinci-002:768 - ] + debug_msg(self.requestline) + debug_msg(self.headers) + if self.path.startswith('/v1/engines') or self.path.startswith('/v1/models'): is_legacy = 'engines' in self.path is_list = self.path in ['/v1/engines', '/v1/models'] - - resp = '' - - if is_legacy and not is_list: # load model + if is_legacy and not is_list: model_name = self.path[self.path.find('/v1/engines/') + len('/v1/engines/'):] - - resp = { - "id": model_name, - "object": "engine", - "owner": "self", - "ready": True, - } - if model_name not in pseudo_model_list + embeddings_model_list + current_model_list: # Real model only - # No args. Maybe it works anyways! - # TODO: hack some heuristics into args for better results - - shared.model_name = model_name - unload_model() - - model_settings = get_model_settings_from_yamls(shared.model_name) - shared.settings.update(model_settings) - update_model_parameters(model_settings, initial=True) - - if shared.settings['mode'] != 'instruct': - shared.settings['instruction_template'] = None - - shared.model, shared.tokenizer = load_model(shared.model_name) - - if not shared.model: # load failed. - shared.model_name = "None" - resp['id'] = "None" - resp['ready'] = False - + resp = OAImodels.load_model(model_name) elif is_list: - # TODO: Lora's? - available_model_list = get_available_models() - all_model_list = current_model_list + embeddings_model_list + pseudo_model_list + available_model_list - - models = {} - - if is_legacy: - models = [{ "id": id, "object": "engine", "owner": "user", "ready": True } for id in all_model_list ] - if not shared.model: - models[0]['ready'] = False - else: - models = [{ "id": id, "object": "model", "owned_by": "user", "permission": [] } for id in all_model_list ] - - resp = { - "object": "list", - "data": models, - } - + resp = OAImodels.list_models(is_legacy) else: - the_model_name = self.path[len('/v1/models/'):] - resp = { - "id": the_model_name, - "object": "model", - "owned_by": "user", - "permission": [] - } + model_name = self.path[len('/v1/models/'):] + resp = OAImodels.model_info() - self.send_response(200) - self.send_access_control_headers() - self.send_header('Content-Type', 'application/json') - self.end_headers() - response = json.dumps(resp) - self.wfile.write(response.encode('utf-8')) + self.return_json(resp) elif '/billing/usage' in self.path: - # Ex. /v1/dashboard/billing/usage?start_date=2023-05-01&end_date=2023-05-31 - self.send_response(200) - self.send_access_control_headers() - self.send_header('Content-Type', 'application/json') - self.end_headers() - - response = json.dumps({ - "total_usage": 0, - }) - self.wfile.write(response.encode('utf-8')) + # Ex. /v1/dashboard/billing/usage?start_date=2023-05-01&end_date=2023-05-31 + self.return_json({"total_usage": 0}, no_debug=True) else: self.send_error(404) + @openai_error_handler def do_POST(self): - if debug: - print(self.headers) # did you know... python-openai sends your linux kernel & python version? + debug_msg(self.requestline) + debug_msg(self.headers) + content_length = int(self.headers['Content-Length']) body = json.loads(self.rfile.read(content_length).decode('utf-8')) - if debug: - print(body) + debug_msg(body) if '/completions' in self.path or '/generate' in self.path: @@ -255,621 +146,109 @@ class Handler(BaseHTTPRequestHandler): return is_legacy = '/generate' in self.path - is_chat_request = 'chat' in self.path - resp_list = 'data' if is_legacy else 'choices' + is_streaming = body.get('stream', False) - # XXX model is ignored for now - # model = body.get('model', shared.model_name) # ignored, use existing for now - model = shared.model_name - created_time = int(time.time()) - - cmpl_id = "chatcmpl-%d" % (created_time) if is_chat_request else "conv-%d" % (created_time) - - # Request Parameters - # Try to use openai defaults or map them to something with the same intent - req_params = default_req_params.copy() - stopping_strings = [] - - if 'stop' in body: - if isinstance(body['stop'], str): - stopping_strings.extend([body['stop']]) - elif isinstance(body['stop'], list): - stopping_strings.extend(body['stop']) - - truncation_length = default(shared.settings, 'truncation_length', 2048) - truncation_length = clamp(default(body, 'truncation_length', truncation_length), 1, truncation_length) - - default_max_tokens = truncation_length if is_chat_request else 16 # completions default, chat default is 'inf' so we need to cap it. - - max_tokens_str = 'length' if is_legacy else 'max_tokens' - max_tokens = default(body, max_tokens_str, default(shared.settings, 'max_new_tokens', default_max_tokens)) - # if the user assumes OpenAI, the max_tokens is way too large - try to ignore it unless it's small enough - - req_params['max_new_tokens'] = max_tokens - req_params['truncation_length'] = truncation_length - req_params['temperature'] = clamp(default(body, 'temperature', default_req_params['temperature']), 0.001, 1.999) # fixup absolute 0.0 - req_params['top_p'] = clamp(default(body, 'top_p', default_req_params['top_p']), 0.001, 1.0) - req_params['top_k'] = default(body, 'best_of', default_req_params['top_k']) - req_params['suffix'] = default(body, 'suffix', default_req_params['suffix']) - req_params['stream'] = default(body, 'stream', default_req_params['stream']) - req_params['echo'] = default(body, 'echo', default_req_params['echo']) - req_params['seed'] = shared.settings.get('seed', default_req_params['seed']) - req_params['add_bos_token'] = shared.settings.get('add_bos_token', default_req_params['add_bos_token']) - - is_streaming = req_params['stream'] - - self.send_response(200) - self.send_access_control_headers() if is_streaming: - self.send_header('Content-Type', 'text/event-stream') - self.send_header('Cache-Control', 'no-cache') - # self.send_header('Connection', 'keep-alive') - else: - self.send_header('Content-Type', 'application/json') - self.end_headers() + self.start_sse() - token_count = 0 - completion_token_count = 0 - prompt = '' - stream_object_type = '' - object_type = '' - - if is_chat_request: - # Chat Completions - stream_object_type = 'chat.completions.chunk' - object_type = 'chat.completions' - - messages = body['messages'] - - role_formats = { - 'user': 'user: {message}\n', - 'assistant': 'assistant: {message}\n', - 'system': '{message}', - 'context': 'You are a helpful assistant. Answer as concisely as possible.', - 'prompt': 'assistant:', - } - - # Instruct models can be much better - if shared.settings['instruction_template']: - try: - instruct = yaml.safe_load(open(f"characters/instruction-following/{shared.settings['instruction_template']}.yaml", 'r')) - - template = instruct['turn_template'] - system_message_template = "{message}" - system_message_default = instruct['context'] - bot_start = template.find('<|bot|>') # So far, 100% of instruction templates have this token - user_message_template = template[:bot_start].replace('<|user-message|>', '{message}').replace('<|user|>', instruct['user']) - bot_message_template = template[bot_start:].replace('<|bot-message|>', '{message}').replace('<|bot|>', instruct['bot']) - bot_prompt = bot_message_template[:bot_message_template.find('{message}')].rstrip(' ') + response = [] + if 'chat' in self.path: + response = OAIcompletions.stream_chat_completions(body, is_legacy=is_legacy) + else: + response = OAIcompletions.stream_completions(body, is_legacy=is_legacy) - role_formats = { - 'user': user_message_template, - 'assistant': bot_message_template, - 'system': system_message_template, - 'context': system_message_default, - 'prompt': bot_prompt, - } + for resp in response: + self.send_sse(resp) - if 'Alpaca' in shared.settings['instruction_template']: - stopping_strings.extend(['\n###']) - elif instruct['user']: # WizardLM and some others have no user prompt. - stopping_strings.extend(['\n' + instruct['user'], instruct['user']]) - - if debug: - print(f"Loaded instruction role format: {shared.settings['instruction_template']}") - - except Exception as e: - stopping_strings.extend(['\nuser:']) - - print(f"Exception: When loading characters/instruction-following/{shared.settings['instruction_template']}.yaml: {repr(e)}") - print("Warning: Loaded default instruction-following template for model.") - - else: - stopping_strings.extend(['\nuser:']) - print("Warning: Loaded default instruction-following template for model.") - - system_msgs = [] - chat_msgs = [] - - # You are ChatGPT, a large language model trained by OpenAI. Answer as concisely as possible. Knowledge cutoff: {knowledge_cutoff} Current date: {current_date} - context_msg = role_formats['system'].format(message=role_formats['context']) if role_formats['context'] else '' - if context_msg: - system_msgs.extend([context_msg]) - - # Maybe they sent both? This is not documented in the API, but some clients seem to do this. - if 'prompt' in body: - prompt_msg = role_formats['system'].format(message=body['prompt']) - system_msgs.extend([prompt_msg]) - - for m in messages: - role = m['role'] - content = m['content'] - msg = role_formats[role].format(message=content) - if role == 'system': - system_msgs.extend([msg]) - else: - chat_msgs.extend([msg]) - - # can't really truncate the system messages - system_msg = '\n'.join(system_msgs) - if system_msg and system_msg[-1] != '\n': - system_msg = system_msg + '\n' - - system_token_count = len(encode(system_msg)[0]) - remaining_tokens = truncation_length - system_token_count - chat_msg = '' - - while chat_msgs: - new_msg = chat_msgs.pop() - new_size = len(encode(new_msg)[0]) - if new_size <= remaining_tokens: - chat_msg = new_msg + chat_msg - remaining_tokens -= new_size - else: - print(f"Warning: too many messages for context size, dropping {len(chat_msgs) + 1} oldest message(s).") - break - - prompt = system_msg + chat_msg + role_formats['prompt'] - - token_count = len(encode(prompt)[0]) + self.end_sse() else: - # Text Completions - stream_object_type = 'text_completion.chunk' - object_type = 'text_completion' - - # ... encoded as a string, array of strings, array of tokens, or array of token arrays. - if is_legacy: - prompt = body['context'] # Older engines.generate API + response = '' + if 'chat' in self.path: + response = OAIcompletions.chat_completions(body, is_legacy=is_legacy) else: - prompt = body['prompt'] # XXX this can be different types + response = OAIcompletions.completions(body, is_legacy=is_legacy) - if isinstance(prompt, list): - self.openai_error("API Batched generation not yet supported.") - return - - token_count = len(encode(prompt)[0]) - if token_count >= truncation_length: - new_len = int(len(prompt) * shared.settings['truncation_length'] / token_count) - prompt = prompt[-new_len:] - new_token_count = len(encode(prompt)[0]) - print(f"Warning: truncating prompt to {new_len} characters, was {token_count} tokens. Now: {new_token_count} tokens.") - token_count = new_token_count - - if truncation_length - token_count < req_params['max_new_tokens']: - print(f"Warning: Ignoring max_new_tokens ({req_params['max_new_tokens']}), too large for the remaining context. Remaining tokens: {truncation_length - token_count}") - req_params['max_new_tokens'] = truncation_length - token_count - print(f"Warning: Set max_new_tokens = {req_params['max_new_tokens']}") - - if is_streaming: - # begin streaming - chunk = { - "id": cmpl_id, - "object": stream_object_type, - "created": created_time, - "model": shared.model_name, - resp_list: [{ - "index": 0, - "finish_reason": None, - }], - } - - if stream_object_type == 'text_completion.chunk': - chunk[resp_list][0]["text"] = "" - else: - # So yeah... do both methods? delta and messages. - chunk[resp_list][0]["message"] = {'role': 'assistant', 'content': ''} - chunk[resp_list][0]["delta"] = {'role': 'assistant', 'content': ''} - - response = 'data: ' + json.dumps(chunk) + '\r\n\r\n' - self.wfile.write(response.encode('utf-8')) - - # generate reply ####################################### - if debug: - print({'prompt': prompt, 'req_params': req_params}) - generator = generate_reply(prompt, req_params, stopping_strings=stopping_strings, is_chat=False) - - answer = '' - seen_content = '' - longest_stop_len = max([len(x) for x in stopping_strings] + [0]) - - for a in generator: - answer = a - - stop_string_found = False - len_seen = len(seen_content) - search_start = max(len_seen - longest_stop_len, 0) - - for string in stopping_strings: - idx = answer.find(string, search_start) - if idx != -1: - answer = answer[:idx] # clip it. - stop_string_found = True - - if stop_string_found: - break - - # If something like "\nYo" is generated just before "\nYou:" - # is completed, buffer and generate more, don't send it - buffer_and_continue = False - - for string in stopping_strings: - for j in range(len(string) - 1, 0, -1): - if answer[-j:] == string[:j]: - buffer_and_continue = True - break - else: - continue - break - - if buffer_and_continue: - continue - - if is_streaming: - # Streaming - new_content = answer[len_seen:] - - if not new_content or chr(0xfffd) in new_content: # partial unicode character, don't send it yet. - continue - - seen_content = answer - chunk = { - "id": cmpl_id, - "object": stream_object_type, - "created": created_time, - "model": shared.model_name, - resp_list: [{ - "index": 0, - "finish_reason": None, - }], - } - - # strip extra leading space off new generated content - if len_seen == 0 and new_content[0] == ' ': - new_content = new_content[1:] - - if stream_object_type == 'text_completion.chunk': - chunk[resp_list][0]['text'] = new_content - else: - # So yeah... do both methods? delta and messages. - chunk[resp_list][0]['message'] = {'content': new_content} - chunk[resp_list][0]['delta'] = {'content': new_content} - response = 'data: ' + json.dumps(chunk) + '\r\n\r\n' - self.wfile.write(response.encode('utf-8')) - completion_token_count += len(encode(new_content)[0]) - - if is_streaming: - chunk = { - "id": cmpl_id, - "object": stream_object_type, - "created": created_time, - "model": model, # TODO: add Lora info? - resp_list: [{ - "index": 0, - "finish_reason": "stop", - }], - "usage": { - "prompt_tokens": token_count, - "completion_tokens": completion_token_count, - "total_tokens": token_count + completion_token_count - } - } - if stream_object_type == 'text_completion.chunk': - chunk[resp_list][0]['text'] = '' - else: - # So yeah... do both methods? delta and messages. - chunk[resp_list][0]['message'] = {'content': ''} - chunk[resp_list][0]['delta'] = {'content': ''} - - response = 'data: ' + json.dumps(chunk) + '\r\n\r\ndata: [DONE]\r\n\r\n' - self.wfile.write(response.encode('utf-8')) - # Finished if streaming. - if debug: - if answer and answer[0] == ' ': - answer = answer[1:] - print({'answer': answer}, chunk) - return - - # strip extra leading space off new generated content - if answer and answer[0] == ' ': - answer = answer[1:] - - if debug: - print({'response': answer}) - - completion_token_count = len(encode(answer)[0]) - stop_reason = "stop" - if token_count + completion_token_count >= truncation_length: - stop_reason = "length" - - resp = { - "id": cmpl_id, - "object": object_type, - "created": created_time, - "model": model, # TODO: add Lora info? - resp_list: [{ - "index": 0, - "finish_reason": stop_reason, - }], - "usage": { - "prompt_tokens": token_count, - "completion_tokens": completion_token_count, - "total_tokens": token_count + completion_token_count - } - } - - if is_chat_request: - resp[resp_list][0]["message"] = {"role": "assistant", "content": answer} - else: - resp[resp_list][0]["text"] = answer - - response = json.dumps(resp) - self.wfile.write(response.encode('utf-8')) + self.return_json(response) elif '/edits' in self.path: + # deprecated + if not shared.model: self.openai_error("No model loaded.") return - self.send_response(200) - self.send_access_control_headers() - self.send_header('Content-Type', 'application/json') - self.end_headers() + req_params = get_default_req_params() - created_time = int(time.time()) - - # Using Alpaca format, this may work with other models too. instruction = body['instruction'] input = body.get('input', '') + temperature = clamp(default(body, 'temperature', req_params['temperature']), 0.001, 1.999) # fixup absolute 0.0 + top_p = clamp(default(body, 'top_p', req_params['top_p']), 0.001, 1.0) - # Request parameters - req_params = default_req_params.copy() - stopping_strings = [] + response = OAIedits.edits(instruction, input, temperature, top_p) - # Alpaca is verbose so a good default prompt - default_template = ( - "Below is an instruction that describes a task, paired with an input that provides further context. " - "Write a response that appropriately completes the request.\n\n" - "### Instruction:\n{instruction}\n\n### Input:\n{input}\n\n### Response:\n" - ) - - instruction_template = default_template - - # Use the special instruction/input/response template for anything trained like Alpaca - if shared.settings['instruction_template']: - if 'Alpaca' in shared.settings['instruction_template']: - stopping_strings.extend(['\n###']) - else: - try: - instruct = yaml.safe_load(open(f"characters/instruction-following/{shared.settings['instruction_template']}.yaml", 'r')) - - template = instruct['turn_template'] - template = template\ - .replace('<|user|>', instruct.get('user', ''))\ - .replace('<|bot|>', instruct.get('bot', ''))\ - .replace('<|user-message|>', '{instruction}\n{input}') - - instruction_template = instruct.get('context', '') + template[:template.find('<|bot-message|>')].rstrip(' ') - if instruct['user']: - stopping_strings.extend(['\n' + instruct['user'], instruct['user'] ]) - - except Exception as e: - instruction_template = default_template - print(f"Exception: When loading characters/instruction-following/{shared.settings['instruction_template']}.yaml: {repr(e)}") - print("Warning: Loaded default instruction-following template (Alpaca) for model.") - else: - stopping_strings.extend(['\n###']) - print("Warning: Loaded default instruction-following template (Alpaca) for model.") - - - edit_task = instruction_template.format(instruction=instruction, input=input) - - truncation_length = default(shared.settings, 'truncation_length', 2048) - token_count = len(encode(edit_task)[0]) - max_tokens = truncation_length - token_count - - req_params['max_new_tokens'] = max_tokens - req_params['truncation_length'] = truncation_length - req_params['temperature'] = clamp(default(body, 'temperature', default_req_params['temperature']), 0.001, 1.999) # fixup absolute 0.0 - req_params['top_p'] = clamp(default(body, 'top_p', default_req_params['top_p']), 0.001, 1.0) - req_params['seed'] = shared.settings.get('seed', default_req_params['seed']) - req_params['add_bos_token'] = shared.settings.get('add_bos_token', default_req_params['add_bos_token']) - - if debug: - print({'edit_template': edit_task, 'req_params': req_params, 'token_count': token_count}) - - generator = generate_reply(edit_task, req_params, stopping_strings=stopping_strings, is_chat=False) - - longest_stop_len = max([len(x) for x in stopping_strings] + [0]) - answer = '' - seen_content = '' - for a in generator: - answer = a - - stop_string_found = False - len_seen = len(seen_content) - search_start = max(len_seen - longest_stop_len, 0) - - for string in stopping_strings: - idx = answer.find(string, search_start) - if idx != -1: - answer = answer[:idx] # clip it. - stop_string_found = True - - if stop_string_found: - break - - - # some reply's have an extra leading space to fit the instruction template, just clip it off from the reply. - if edit_task[-1] != '\n' and answer and answer[0] == ' ': - answer = answer[1:] - - completion_token_count = len(encode(answer)[0]) - - resp = { - "object": "edit", - "created": created_time, - "choices": [{ - "text": answer, - "index": 0, - }], - "usage": { - "prompt_tokens": token_count, - "completion_tokens": completion_token_count, - "total_tokens": token_count + completion_token_count - } - } - - if debug: - print({'answer': answer, 'completion_token_count': completion_token_count}) - - response = json.dumps(resp) - self.wfile.write(response.encode('utf-8')) + self.return_json(response) elif '/images/generations' in self.path and 'SD_WEBUI_URL' in os.environ: - # Stable Diffusion callout wrapper for txt2img - # Low effort implementation for compatibility. With only "prompt" being passed and assuming DALL-E - # the results will be limited and likely poor. SD has hundreds of models and dozens of settings. - # If you want high quality tailored results you should just use the Stable Diffusion API directly. - # it's too general an API to try and shape the result with specific tags like "masterpiece", etc, - # Will probably work best with the stock SD models. - # SD configuration is beyond the scope of this API. - # At this point I will not add the edits and variations endpoints (ie. img2img) because they - # require changing the form data handling to accept multipart form data, also to properly support - # url return types will require file management and a web serving files... Perhaps later! - - self.send_response(200) - self.send_access_control_headers() - self.send_header('Content-Type', 'application/json') - self.end_headers() - - width, height = [ int(x) for x in default(body, 'size', '1024x1024').split('x') ] # ignore the restrictions on size + prompt = body['prompt'] + size = default(body, 'size', '1024x1024') response_format = default(body, 'response_format', 'url') # or b64_json + n = default(body, 'n', 1) # ignore the batch limits of max 10 + + response = OAIimages.generations(prompt=prompt, size=size, response_format=response_format, n=n) + + self.return_json(response, no_debug=True) + + elif '/embeddings' in self.path: + encoding_format = body.get('encoding_format', '') + + input = body.get('input', body.get('text', '')) + if not input: + raise InvalidRequestError("Missing required argument input", params='input') - payload = { - 'prompt': body['prompt'], # ignore prompt limit of 1000 characters - 'width': width, - 'height': height, - 'batch_size': default(body, 'n', 1) # ignore the batch limits of max 10 - } - - resp = { - 'created': int(time.time()), - 'data': [] - } - - # TODO: support SD_WEBUI_AUTH username:password pair. - sd_url = f"{os.environ['SD_WEBUI_URL']}/sdapi/v1/txt2img" - - response = requests.post(url=sd_url, json=payload) - r = response.json() - # r['parameters']... - for b64_json in r['images']: - if response_format == 'b64_json': - resp['data'].extend([{'b64_json': b64_json}]) - else: - resp['data'].extend([{'url': f'data:image/png;base64,{b64_json}'}]) # yeah it's lazy. requests.get() will not work with this - - response = json.dumps(resp) - self.wfile.write(response.encode('utf-8')) - - elif '/embeddings' in self.path and embedding_model is not None: - self.send_response(200) - self.send_access_control_headers() - self.send_header('Content-Type', 'application/json') - self.end_headers() - - input = body['input'] if 'input' in body else body['text'] if type(input) is str: input = [input] - embeddings = embedding_model.encode(input).tolist() + response = OAIembeddings.embeddings(input, encoding_format) - def enc_emb(emb): - # If base64 is specified, encode. Otherwise, do nothing. - if body.get("encoding_format", "") == "base64": - return float_list_to_base64(emb) - else: - return emb - data = [{"object": "embedding", "embedding": enc_emb(emb), "index": n} for n, emb in enumerate(embeddings)] - - response = json.dumps({ - "object": "list", - "data": data, - "model": st_model, # return the real model - "usage": { - "prompt_tokens": 0, - "total_tokens": 0, - } - }) - - if debug: - print(f"Embeddings return size: {len(embeddings[0])}, number: {len(embeddings)}") - self.wfile.write(response.encode('utf-8')) + self.return_json(response, no_debug=True) elif '/moderations' in self.path: - # for now do nothing, just don't error. - self.send_response(200) - self.send_access_control_headers() - self.send_header('Content-Type', 'application/json') - self.end_headers() + input = body['input'] + if not input: + raise InvalidRequestError("Missing required argument input", params='input') - response = json.dumps({ - "id": "modr-5MWoLO", - "model": "text-moderation-001", - "results": [{ - "categories": { - "hate": False, - "hate/threatening": False, - "self-harm": False, - "sexual": False, - "sexual/minors": False, - "violence": False, - "violence/graphic": False - }, - "category_scores": { - "hate": 0.0, - "hate/threatening": 0.0, - "self-harm": 0.0, - "sexual": 0.0, - "sexual/minors": 0.0, - "violence": 0.0, - "violence/graphic": 0.0 - }, - "flagged": False - }] - }) - self.wfile.write(response.encode('utf-8')) + response = OAImoderations.moderations(input) + + self.return_json(response, no_debug=True) elif self.path == '/api/v1/token-count': # NOT STANDARD. lifted from the api extension, but it's still very useful to calculate tokenized length client side. - self.send_response(200) - self.send_access_control_headers() - self.send_header('Content-Type', 'application/json') - self.end_headers() + response = token_count(body['prompt']) + + self.return_json(response, no_debug=True) - tokens = encode(body['prompt'])[0] - response = json.dumps({ - 'results': [{ - 'tokens': len(tokens) - }] - }) - self.wfile.write(response.encode('utf-8')) + elif self.path == '/api/v1/token/encode': + # NOT STANDARD. needed to support logit_bias, logprobs and token arrays for native models + encoding_format = body.get('encoding_format', '') + + response = token_encode(body['input'], encoding_format) + + self.return_json(response, no_debug=True) + + elif self.path == '/api/v1/token/decode': + # NOT STANDARD. needed to support logit_bias, logprobs and token arrays for native models + encoding_format = body.get('encoding_format', '') + + response = token_decode(body['input'], encoding_format) + + self.return_json(response, no_debug=True) else: - print(self.path, self.headers) self.send_error(404) def run_server(): - global embedding_model - try: - embedding_model = SentenceTransformer(st_model) - print(f"\nLoaded embedding model: {st_model}, max sequence length: {embedding_model.max_seq_length}") - except: - print(f"\nFailed to load embedding model: {st_model}") - pass - server_addr = ('0.0.0.0' if shared.args.listen else '127.0.0.1', params['port']) server = ThreadingHTTPServer(server_addr, Handler) if shared.args.share: @@ -881,7 +260,7 @@ def run_server(): print('You should install flask_cloudflared manually') else: print(f'Starting OpenAI compatible api:\nOPENAI_API_BASE=http://{server_addr[0]}:{server_addr[1]}/v1') - + server.serve_forever() diff --git a/extensions/openai/tokens.py b/extensions/openai/tokens.py new file mode 100644 index 00000000..9f0208d3 --- /dev/null +++ b/extensions/openai/tokens.py @@ -0,0 +1,37 @@ +from extensions.openai.utils import float_list_to_base64 +from modules.text_generation import encode, decode + +def token_count(prompt): + tokens = encode(prompt)[0] + + return { + 'results': [{ + 'tokens': len(tokens) + }] + } + + +def token_encode(input, encoding_format = ''): + #if isinstance(input, list): + tokens = encode(input)[0] + + return { + 'results': [{ + 'encoding_format': encoding_format, + 'tokens': float_list_to_base64(tokens) if encoding_format == "base64" else tokens, + 'length': len(tokens), + }] + } + + +def token_decode(tokens, encoding_format): + #if isinstance(input, list): +# if encoding_format == "base64": +# tokens = base64_to_float_list(tokens) + output = decode(tokens)[0] + + return { + 'results': [{ + 'text': output + }] + } diff --git a/extensions/openai/utils.py b/extensions/openai/utils.py new file mode 100644 index 00000000..d01ec14d --- /dev/null +++ b/extensions/openai/utils.py @@ -0,0 +1,26 @@ +import os +import base64 +import numpy as np + +def float_list_to_base64(float_list): + # Convert the list to a float32 array that the OpenAPI client expects + float_array = np.array(float_list, dtype="float32") + + # Get raw bytes + bytes_array = float_array.tobytes() + + # Encode bytes into base64 + encoded_bytes = base64.b64encode(bytes_array) + + # Turn raw base64 encoded bytes into ASCII + ascii_string = encoded_bytes.decode('ascii') + return ascii_string + +def end_line(s): + if s and s[-1] != '\n': + s = s + '\n' + return s + +def debug_msg(*args, **kwargs): + if 'OPENEDAI_DEBUG' in os.environ: + print(*args, **kwargs) \ No newline at end of file From 3e9da5a27ceb2a17941ff7c287f1382bd6ab9124 Mon Sep 17 00:00:00 2001 From: Ricardo Pinto Date: Tue, 11 Jul 2023 22:52:16 +0100 Subject: [PATCH 24/54] Changed FormComponent to IOComponent (#3017) Co-authored-by: Ricardo Pinto <1-ricardo.pinto@users.noreply.gitlab.cognitage.com> --- modules/ui.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ui.py b/modules/ui.py index 35049533..9fea2880 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -159,7 +159,7 @@ def apply_interface_values(state, use_persistent=False): return [state[k] if k in state else gr.update() for k in elements] -class ToolButton(gr.Button, gr.components.FormComponent): +class ToolButton(gr.Button, gr.components.IOComponent): """Small button with single emoji as text, fits inside gradio forms""" def __init__(self, **kwargs): From 3778816b8d921f04ab8b362e617dc5dd14b635a2 Mon Sep 17 00:00:00 2001 From: matatonic <73265741+matatonic@users.noreply.github.com> Date: Tue, 11 Jul 2023 17:53:48 -0400 Subject: [PATCH 25/54] models/config.yaml: +platypus/gplatty, +longchat, +vicuna-33b, +Redmond-Hermes-Coder, +wizardcoder, +more (#2928) * +platypus/gplatty * +longchat, +vicuna-33b, +Redmond-Hermes-Coder * +wizardcoder * +superplatty * +Godzilla, +WizardLM-V1.1, +rwkv 8k, +wizard-mega fix --------- Co-authored-by: Matthew Ashton --- models/config.yaml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/models/config.yaml b/models/config.yaml index d81eac94..37e4273c 100644 --- a/models/config.yaml +++ b/models/config.yaml @@ -97,6 +97,8 @@ llama-65b-gptq-3bit: .*raven: mode: 'instruct' instruction_template: 'RWKV-Raven' +.*ctx8192: + truncation_length: 8192 .*moss-moon.*sft: mode: 'instruct' instruction_template: 'MOSS' @@ -143,6 +145,7 @@ llama-65b-gptq-3bit: .*wizard.*mega: mode: 'instruct' instruction_template: 'Wizard-Mega' + custom_stopping_strings: '""' .*ziya-: mode: 'instruct' instruction_template: 'Ziya' @@ -243,3 +246,26 @@ TheBloke_WizardLM-30B-GPTQ: .*xgen.*-inst: truncation_length: 8192 instruction_template: 'Vicuna-v0' +.*(platypus|gplatty|superplatty): + mode: 'instruct' + instruction_template: 'Alpaca' +.*longchat: + mode: 'instruct' + instruction_template: 'Vicuna-v1.1' +.*vicuna-33b: + mode: 'instruct' + instruction_template: 'Vicuna-v1.1' +.*redmond-hermes-coder: + mode: 'instruct' + instruction_template: 'Alpaca' + truncation_length: 8192 +.*wizardcoder-15b: + mode: 'instruct' + instruction_template: 'Alpaca' + truncation_length: 8192 +.*wizardlm-.*-v1.1: + mode: 'instruct' + instruction_template: 'Vicuna-v1.1' +.*godzilla: + mode: 'instruct' + instruction_template: 'Alpaca' From 3708de2b1f3344940e307b2487ea4a6c0cdaa63d Mon Sep 17 00:00:00 2001 From: micsthepick <11528421+micsthepick@users.noreply.github.com> Date: Wed, 12 Jul 2023 07:55:46 +1000 Subject: [PATCH 26/54] respect model dir for downloads (#3077) (#3079) --- server.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/server.py b/server.py index 18474d3b..412f13ec 100644 --- a/server.py +++ b/server.py @@ -145,7 +145,13 @@ def download_model_wrapper(repo_id, progress=gr.Progress()): links, sha256, is_lora = downloader.get_download_links_from_huggingface(model, branch, text_only=False) yield ("Getting the output folder") - output_folder = downloader.get_output_folder(model, branch, is_lora) + models_dir = Path(shared.args.model_dir) + + # If the last part of the path is "models", remove it + if models_dir.name.lower() == 'models': + models_dir = models_dir.parent + + output_folder = downloader.get_output_folder(model, branch, is_lora, base_folder=models_dir) if check: progress(0.5) From ab044a5a448151241e587164eab011628a96b74b Mon Sep 17 00:00:00 2001 From: "Salvador E. Tropea" Date: Tue, 11 Jul 2023 19:00:37 -0300 Subject: [PATCH 27/54] Elevenlabs tts fixes (#2959) * [Fixed] Keep setting option for the voice - It was always changed to the first available voice - Also added an error if the selected voice isn't valid * [Fixed] elevenlabs_tts API key handling - The one from the settings wasn't applied - We always got "Enter your API key", even when the settings specified an api_key * [Added] elevenlabs_tts model selection - Now we can also use the "eleven_multilingual_v1" model. Used for anything but english. --- extensions/elevenlabs_tts/script.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/extensions/elevenlabs_tts/script.py b/extensions/elevenlabs_tts/script.py index c5e6174b..a64822ce 100644 --- a/extensions/elevenlabs_tts/script.py +++ b/extensions/elevenlabs_tts/script.py @@ -6,6 +6,7 @@ import gradio as gr from modules import chat, shared from modules.utils import gradio +from modules.logging_colors import logger params = { 'activate': True, @@ -13,10 +14,12 @@ params = { 'selected_voice': 'None', 'autoplay': False, 'show_text': True, + 'model': 'eleven_monolingual_v1', } voices = None wav_idx = 0 +LANG_MODELS = ['eleven_monolingual_v1', 'eleven_multilingual_v1'] def update_api_key(key): @@ -108,7 +111,7 @@ def output_modifier(string): output_file = Path(f'extensions/elevenlabs_tts/outputs/{wav_idx:06d}.mp3'.format(wav_idx)) print(f'Outputting audio to {str(output_file)}') try: - audio = elevenlabs.generate(text=string, voice=params['selected_voice'], model="eleven_monolingual_v1") + audio = elevenlabs.generate(text=string, voice=params['selected_voice'], model=params['model']) elevenlabs.save(audio, str(output_file)) autoplay = 'autoplay' if params['autoplay'] else '' @@ -132,7 +135,12 @@ def ui(): global voices if not voices: voices = refresh_voices() - params['selected_voice'] = voices[0] + selected = params['selected_voice'] + if selected == 'None': + params['selected_voice'] = voices[0] + elif selected not in voices: + logger.error(f'Selected voice {selected} not available, switching to {voices[0]}') + params['selected_voice'] = voices[0] # Gradio elements with gr.Row(): @@ -145,7 +153,14 @@ def ui(): refresh = gr.Button(value='Refresh') with gr.Row(): - api_key = gr.Textbox(placeholder="Enter your API key.", label='API Key') + if params['api_key']: + api_key = gr.Textbox(value=params['api_key'], label='API Key') + update_api_key(params['api_key']) + else: + api_key = gr.Textbox(placeholder="Enter your API key.", label='API Key') + + with gr.Row(): + model = gr.Dropdown(value=params['model'], choices=LANG_MODELS, label='Language model') with gr.Row(): convert = gr.Button('Permanently replace audios with the message texts') @@ -175,6 +190,7 @@ def ui(): activate.change(lambda x: params.update({'activate': x}), activate, None) voice.change(lambda x: params.update({'selected_voice': x}), voice, None) api_key.change(update_api_key, api_key, None) + model.change(lambda x: params.update({'model': x}), model, None) # connect.click(check_valid_api, [], connection_status) refresh.click(refresh_voices_dd, [], voice) # Event functions to update the parameters in the backend From 1fc0b5041e3c903263b68065ce474f4e4e3a658a Mon Sep 17 00:00:00 2001 From: Juliano Henriquez Date: Tue, 11 Jul 2023 18:02:49 -0400 Subject: [PATCH 28/54] substitu superboog Beatiful Soup Parser (#2996) * add lxml to requirments add lxml to requirments * Change Beaitful Soup Parser "lxml" parser which might be more tolerant of certain kinds of parsing errors than "html.parser" and quicker at the same time. --- extensions/superbooga/requirements.txt | 1 + extensions/superbooga/script.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/extensions/superbooga/requirements.txt b/extensions/superbooga/requirements.txt index dd2cbde6..4d5a95a4 100644 --- a/extensions/superbooga/requirements.txt +++ b/extensions/superbooga/requirements.txt @@ -2,3 +2,4 @@ beautifulsoup4==4.12.2 chromadb==0.3.18 posthog==2.4.2 sentence_transformers==2.2.2 +lxml diff --git a/extensions/superbooga/script.py b/extensions/superbooga/script.py index c0d3f8eb..f67a956e 100644 --- a/extensions/superbooga/script.py +++ b/extensions/superbooga/script.py @@ -69,7 +69,7 @@ def feed_url_into_collector(urls, chunk_len, chunk_sep, strong_cleanup, threads) cumulative += 'Processing the HTML sources...' yield cumulative for content in contents: - soup = BeautifulSoup(content, features="html.parser") + soup = BeautifulSoup(content, features="lxml") for script in soup(["script", "style"]): script.extract() From 37bffb2e1a4bb63f440e0aef3d7ee57a1a3d02bd Mon Sep 17 00:00:00 2001 From: Keith Kjer Date: Tue, 11 Jul 2023 15:04:15 -0700 Subject: [PATCH 29/54] Add reference to new pipeline in multimodal readme (#2947) --- extensions/multimodal/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/extensions/multimodal/README.md b/extensions/multimodal/README.md index 0f515ae6..10bbc7f5 100644 --- a/extensions/multimodal/README.md +++ b/extensions/multimodal/README.md @@ -38,6 +38,8 @@ As of now, the following multimodal pipelines are supported: |[LLaVA 7B](https://github.com/haotian-liu/LLaVA)|`llava-7b`|[LLaVA 7B](https://huggingface.co/wojtab/llava-7b-v0-4bit-128g)|GPTQ 4-bit quant, old CUDA|built-in| |[MiniGPT-4 7B](https://github.com/Vision-CAIR/MiniGPT-4)|`minigpt4-7b`|[Vicuna v0 7B](https://huggingface.co/TheBloke/vicuna-7B-GPTQ-4bit-128g)|GPTQ 4-bit quant, new format|[Wojtab/minigpt-4-pipeline](https://github.com/Wojtab/minigpt-4-pipeline)| |[MiniGPT-4 13B](https://github.com/Vision-CAIR/MiniGPT-4)|`minigpt4-13b`|[Vicuna v0 13B](https://huggingface.co/anon8231489123/vicuna-13b-GPTQ-4bit-128g)|GPTQ 4-bit quant, old CUDA|[Wojtab/minigpt-4-pipeline](https://github.com/Wojtab/minigpt-4-pipeline)| +|[InstructBLIP 7B](https://github.com/salesforce/LAVIS/tree/main/projects/instructblip)|`instructblip-7b`|[Vicuna v1.1 7B](https://huggingface.co/TheBloke/vicuna-7B-1.1-GPTQ-4bit-128g)|GPTQ 4-bit quant|[kjerk/instructblip-pipeline](https://github.com/kjerk/instructblip-pipeline)| +|[InstructBLIP 13B](https://github.com/salesforce/LAVIS/tree/main/projects/instructblip)|`instructblip-13b`|[Vicuna v1.1 13B](https://huggingface.co/TheBloke/vicuna-13B-1.1-GPTQ-4bit-128g)|GPTQ 4-bit quant|[kjerk/instructblip-pipeline](https://github.com/kjerk/instructblip-pipeline)| Some pipelines could support different LLMs but do note that while it might work, it isn't a supported configuration. From a12dae51b92e023c85599eb03f83c789b9a414be Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 11 Jul 2023 18:29:08 -0700 Subject: [PATCH 30/54] Bump bitsandbytes --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 93d07060..681be6d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,7 +18,7 @@ tqdm scipy transformers==4.30.2 git+https://github.com/huggingface/peft@03eb378eb914fbee709ff7c86ba5b1d033b89524 -bitsandbytes==0.39.1; platform_system != "Windows" +bitsandbytes==0.40.0; platform_system != "Windows" https://github.com/jllllll/bitsandbytes-windows-webui/releases/download/wheels/bitsandbytes-0.40.0-py3-none-win_amd64.whl; platform_system == "Windows" llama-cpp-python==0.1.70; platform_system != "Windows" https://github.com/abetlen/llama-cpp-python/releases/download/v0.1.70/llama_cpp_python-0.1.70-cp310-cp310-win_amd64.whl; platform_system == "Windows" From bfafd07f442878a48edfbf29779fe0b2b00c820c Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 11 Jul 2023 18:29:20 -0700 Subject: [PATCH 31/54] Change a message --- server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server.py b/server.py index 412f13ec..53948bd5 100644 --- a/server.py +++ b/server.py @@ -54,7 +54,7 @@ from modules.utils import gradio def load_model_wrapper(selected_model, loader, autoload=False): if not autoload: - yield f"The settings for {selected_model} have been updated.\nClick on \"Load the model\" to load it." + yield f"The settings for {selected_model} have been updated.\nClick on \"Load\" to load it." return if selected_model == 'None': From e3810dff40ff89d3989253f6fa378946b5e640a8 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 11 Jul 2023 18:49:06 -0700 Subject: [PATCH 32/54] Style changes --- modules/training.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/modules/training.py b/modules/training.py index cdf7c591..29383e7f 100644 --- a/modules/training.py +++ b/modules/training.py @@ -1,19 +1,17 @@ import json import math import random +import shutil import sys import threading import time import traceback +from datetime import datetime from pathlib import Path import gradio as gr import torch import transformers - -import shutil -from datetime import datetime - from datasets import Dataset, load_dataset from peft import ( LoraConfig, @@ -223,7 +221,7 @@ def backup_adapter(input_folder): creation_date_str = creation_date.strftime("Backup-%Y-%m-%d") # Create the new subfolder - subfolder_path = Path(f"{input_folder}/{creation_date_str}") + subfolder_path = Path(f"{input_folder}/{creation_date_str}") subfolder_path.mkdir(parents=True, exist_ok=True) # Check if the file already exists in the subfolder @@ -240,6 +238,7 @@ def backup_adapter(input_folder): except Exception as e: print("An error occurred in backup_adapter:", str(e)) + def calc_trainable_parameters(model): trainable_params = 0 all_param = 0 @@ -252,8 +251,8 @@ def calc_trainable_parameters(model): all_param += num_params if param.requires_grad: trainable_params += num_params - - return trainable_params,all_param + + return trainable_params, all_param def do_train(lora_name: str, always_override: bool, save_steps: int, micro_batch_size: int, batch_size: int, epochs: int, learning_rate: str, lr_scheduler_type: str, lora_rank: int, lora_alpha: int, lora_dropout: float, cutoff_len: int, dataset: str, eval_dataset: str, format: str, eval_steps: int, raw_text_file: str, overlap_len: int, newline_favor_len: int, higher_rank_limit: bool, warmup_steps: int, optimizer: str, hard_cut_string: str, train_only_after: str, stop_at_loss: float): @@ -559,10 +558,9 @@ def do_train(lora_name: str, always_override: bool, save_steps: int, micro_batch yield "Starting..." lora_trainable_param, lora_all_param = calc_trainable_parameters(lora_model) - - if lora_all_param>0: - print(f"Trainable params: {lora_trainable_param:,d} ({100 * lora_trainable_param / lora_all_param:.4f} %), All params: {lora_all_param:,d} (Model: {model_all_params:,d})") + if lora_all_param > 0: + print(f"Trainable params: {lora_trainable_param:,d} ({100 * lora_trainable_param / lora_all_param:.4f} %), All params: {lora_all_param:,d} (Model: {model_all_params:,d})") train_log.update({"base_model_name": shared.model_name}) train_log.update({"base_model_class": shared.model.__class__.__name__}) From 324e45b84886779ac2b323e737358e796d3b687c Mon Sep 17 00:00:00 2001 From: "Salvador E. Tropea" Date: Tue, 11 Jul 2023 23:27:38 -0300 Subject: [PATCH 33/54] [Fixed] wbits and groupsize values from model not shown (#2977) --- modules/models_settings.py | 5 ++++- server.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/modules/models_settings.py b/modules/models_settings.py index 0207e7de..3f37e48d 100644 --- a/modules/models_settings.py +++ b/modules/models_settings.py @@ -99,7 +99,10 @@ def apply_model_settings_to_state(model, state): for k in model_settings: if k in state: - state[k] = model_settings[k] + if k in ['wbits', 'groupsize']: + state[k] = str(model_settings[k]) + else: + state[k] = model_settings[k] return state diff --git a/server.py b/server.py index 53948bd5..489936e5 100644 --- a/server.py +++ b/server.py @@ -224,8 +224,8 @@ def create_model_menus(): shared.gradio['n_batch'] = gr.Slider(label="n_batch", minimum=1, maximum=2048, value=shared.args.n_batch) shared.gradio['n_gpu_layers'] = gr.Slider(label="n-gpu-layers", minimum=0, maximum=128, value=shared.args.n_gpu_layers) shared.gradio['n_ctx'] = gr.Slider(minimum=0, maximum=16384, step=256, label="n_ctx", value=shared.args.n_ctx) - shared.gradio['wbits'] = gr.Dropdown(label="wbits", choices=["None", 1, 2, 3, 4, 8], value=shared.args.wbits if shared.args.wbits > 0 else "None") - shared.gradio['groupsize'] = gr.Dropdown(label="groupsize", choices=["None", 32, 64, 128, 1024], value=shared.args.groupsize if shared.args.groupsize > 0 else "None") + shared.gradio['wbits'] = gr.Dropdown(label="wbits", choices=["None", 1, 2, 3, 4, 8], value=str(shared.args.wbits) if shared.args.wbits > 0 else "None") + shared.gradio['groupsize'] = gr.Dropdown(label="groupsize", choices=["None", 32, 64, 128, 1024], value=str(shared.args.groupsize) if shared.args.groupsize > 0 else "None") shared.gradio['model_type'] = gr.Dropdown(label="model_type", choices=["None", "llama", "opt", "gptj"], value=shared.args.model_type or "None") shared.gradio['pre_layer'] = gr.Slider(label="pre_layer", minimum=0, maximum=100, value=shared.args.pre_layer[0] if shared.args.pre_layer is not None else 0) shared.gradio['autogptq_info'] = gr.Markdown('On some systems, AutoGPTQ can be 2x slower than GPTQ-for-LLaMa. You can manually select the GPTQ-for-LLaMa loader above.') From d9fabdde40af766837d3cc7d0758189ab0f6ea8d Mon Sep 17 00:00:00 2001 From: atriantafy Date: Wed, 12 Jul 2023 04:01:03 +0100 Subject: [PATCH 34/54] =?UTF-8?q?Add=20context=5Finstruct=20to=20API.=20Lo?= =?UTF-8?q?ad=20default=20model=20instruction=20template=20=E2=80=A6=20(#2?= =?UTF-8?q?688)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api-examples/api-example-chat-stream.py | 5 ++- api-examples/api-example-chat.py | 3 +- api-examples/api-example-model.py | 50 ++++++++++++------------- api-examples/api-example-stream.py | 2 +- api-examples/api-example.py | 2 +- extensions/api/util.py | 7 +++- 6 files changed, 37 insertions(+), 32 deletions(-) diff --git a/api-examples/api-example-chat-stream.py b/api-examples/api-example-chat-stream.py index 8e37b569..14f6f9d6 100644 --- a/api-examples/api-example-chat-stream.py +++ b/api-examples/api-example-chat-stream.py @@ -23,7 +23,8 @@ async def run(user_input, history): 'history': history, 'mode': 'instruct', # Valid options: 'chat', 'chat-instruct', 'instruct' 'character': 'Example', - 'instruction_template': 'Vicuna-v1.1', + 'instruction_template': 'Vicuna-v1.1', # Will get autodetected if unset + # 'context_instruct': '', # Optional 'your_name': 'You', 'regenerate': False, @@ -34,7 +35,7 @@ async def run(user_input, history): # Generation params. If 'preset' is set to different than 'None', the values # in presets/preset-name.yaml are used instead of the individual numbers. - 'preset': 'None', + 'preset': 'None', 'do_sample': True, 'temperature': 0.7, 'top_p': 0.1, diff --git a/api-examples/api-example-chat.py b/api-examples/api-example-chat.py index 23f2f186..0e155c63 100644 --- a/api-examples/api-example-chat.py +++ b/api-examples/api-example-chat.py @@ -17,7 +17,8 @@ def run(user_input, history): 'history': history, 'mode': 'instruct', # Valid options: 'chat', 'chat-instruct', 'instruct' 'character': 'Example', - 'instruction_template': 'Vicuna-v1.1', + 'instruction_template': 'Vicuna-v1.1', # Will get autodetected if unset + # 'context_instruct': '', # Optional 'your_name': 'You', 'regenerate': False, diff --git a/api-examples/api-example-model.py b/api-examples/api-example-model.py index 1e108a2d..9a61ccb1 100644 --- a/api-examples/api-example-model.py +++ b/api-examples/api-example-model.py @@ -4,8 +4,9 @@ import requests HOST = '0.0.0.0:5000' -def generate(prompt, tokens = 200): - request = { 'prompt': prompt, 'max_new_tokens': tokens } + +def generate(prompt, tokens=200): + request = {'prompt': prompt, 'max_new_tokens': tokens} response = requests.post(f'http://{HOST}/api/v1/generate', json=request) if response.status_code == 200: @@ -23,7 +24,7 @@ def print_basic_model_info(response): print("Model: ", response['result']['model_name']) print("Lora(s): ", response['result']['lora_names']) for setting in basic_settings: - print(setting, "=", response['result']['shared.settings'][setting]) + print(setting, "=", response['result']['shared.settings'][setting]) # model info @@ -75,17 +76,17 @@ def complex_model_load(model): 'rwkv_cuda_on': False, # b&b 4-bit - #'load_in_4bit': False, - #'compute_dtype': 'float16', - #'quant_type': 'nf4', - #'use_double_quant': False, + # 'load_in_4bit': False, + # 'compute_dtype': 'float16', + # 'quant_type': 'nf4', + # 'use_double_quant': False, - #"cpu": false, - #"auto_devices": false, - #"gpu_memory": null, - #"cpu_memory": null, - #"disk": false, - #"disk_cache_dir": "cache", + # "cpu": false, + # "auto_devices": false, + # "gpu_memory": null, + # "cpu_memory": null, + # "disk": false, + # "disk_cache_dir": "cache", }, } @@ -104,26 +105,25 @@ def complex_model_load(model): req['args']['load_in_8bit'] = True elif '-hf' in model or 'fp16' in model: if '7b' in model: - req['args']['bf16'] = True # for 24GB + req['args']['bf16'] = True # for 24GB elif '13b' in model: - req['args']['load_in_8bit'] = True # for 24GB + req['args']['load_in_8bit'] = True # for 24GB elif 'ggml' in model: - #req['args']['threads'] = 16 + # req['args']['threads'] = 16 if '7b' in model: req['args']['n_gpu_layers'] = 100 elif '13b' in model: req['args']['n_gpu_layers'] = 100 elif '30b' in model or '33b' in model: - req['args']['n_gpu_layers'] = 59 # 24GB + req['args']['n_gpu_layers'] = 59 # 24GB elif '65b' in model: - req['args']['n_gpu_layers'] = 42 # 24GB + req['args']['n_gpu_layers'] = 42 # 24GB elif 'rwkv' in model: req['args']['rwkv_cuda_on'] = True if '14b' in model: - req['args']['rwkv_strategy'] = 'cuda f16i8' # 24GB + req['args']['rwkv_strategy'] = 'cuda f16i8' # 24GB else: - req['args']['rwkv_strategy'] = 'cuda f16' # 24GB - + req['args']['rwkv_strategy'] = 'cuda f16' # 24GB return model_api(req) @@ -134,7 +134,7 @@ if __name__ == '__main__': resp = complex_model_load(model) if 'error' in resp: - print (f"❌ {model} FAIL Error: {resp['error']['message']}") + print(f"❌ {model} FAIL Error: {resp['error']['message']}") continue else: print_basic_model_info(resp) @@ -142,12 +142,12 @@ if __name__ == '__main__': ans = generate("0,1,1,2,3,5,8,13,", tokens=2) if '21' in ans: - print (f"✅ {model} PASS ({ans})") + print(f"✅ {model} PASS ({ans})") else: - print (f"❌ {model} FAIL ({ans})") + print(f"❌ {model} FAIL ({ans})") except Exception as e: - print (f"❌ {model} FAIL Exception: {repr(e)}") + print(f"❌ {model} FAIL Exception: {repr(e)}") # 0,1,1,2,3,5,8,13, is the fibonacci sequence, the next number is 21. diff --git a/api-examples/api-example-stream.py b/api-examples/api-example-stream.py index 79a01e4d..1ae5a91c 100644 --- a/api-examples/api-example-stream.py +++ b/api-examples/api-example-stream.py @@ -23,7 +23,7 @@ async def run(context): # Generation params. If 'preset' is set to different than 'None', the values # in presets/preset-name.yaml are used instead of the individual numbers. - 'preset': 'None', + 'preset': 'None', 'do_sample': True, 'temperature': 0.7, 'top_p': 0.1, diff --git a/api-examples/api-example.py b/api-examples/api-example.py index b09823c3..4e45de9e 100644 --- a/api-examples/api-example.py +++ b/api-examples/api-example.py @@ -15,7 +15,7 @@ def run(prompt): # Generation params. If 'preset' is set to different than 'None', the values # in presets/preset-name.yaml are used instead of the individual numbers. - 'preset': 'None', + 'preset': 'None', 'do_sample': True, 'temperature': 0.7, 'top_p': 0.1, diff --git a/extensions/api/util.py b/extensions/api/util.py index a89365ce..a25c7885 100644 --- a/extensions/api/util.py +++ b/extensions/api/util.py @@ -59,7 +59,10 @@ def build_parameters(body, chat=False): if chat: character = body.get('character') - instruction_template = body.get('instruction_template') + instruction_template = body.get('instruction_template', shared.settings['instruction_template']) + if str(instruction_template) == "None": + instruction_template = "Vicuna-v1.1" + name1, name2, _, greeting, context, _ = load_character_memoized(character, str(body.get('your_name', shared.settings['name1'])), shared.settings['name2'], instruct=False) name1_instruct, name2_instruct, _, _, context_instruct, turn_template = load_character_memoized(instruction_template, '', '', instruct=True) generate_params.update({ @@ -72,7 +75,7 @@ def build_parameters(body, chat=False): 'greeting': greeting, 'name1_instruct': name1_instruct, 'name2_instruct': name2_instruct, - 'context_instruct': context_instruct, + 'context_instruct': body.get('context_instruct', context_instruct), 'turn_template': turn_template, 'chat-instruct_command': str(body.get('chat-instruct_command', shared.settings['chat-instruct_command'])), 'history': body.get('history', {'internal': [], 'visible': []}) From d986c17c520509443ab2a8e0e65ad6ca6fab4262 Mon Sep 17 00:00:00 2001 From: Axiom Wolf <30778583+UnskilledWolf@users.noreply.github.com> Date: Wed, 12 Jul 2023 05:10:36 +0200 Subject: [PATCH 35/54] Chat history download creates more detailed file names (#3051) --- modules/chat.py | 20 +++++++++++++++++++- server.py | 2 +- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/modules/chat.py b/modules/chat.py index c0635c23..be524ad0 100644 --- a/modules/chat.py +++ b/modules/chat.py @@ -3,6 +3,7 @@ import copy import functools import json import re +from datetime import datetime from pathlib import Path import gradio as gr @@ -388,8 +389,25 @@ def load_history(file, history): return history +def save_history_at_user_request(history, character, mode): + def make_timestamp_path(character=None): + return f"logs/{character or ''}{'_' if character else ''}{datetime.now().strftime('%Y%m%d-%H%M%S')}.json" + + path = None + if mode in ['chat', 'chat-instruct'] and character not in ['', 'None', None]: + path = make_timestamp_path(character) + else: + # Try to use mode as the file name, otherwise just use the timestamp + try: + path = make_timestamp_path(mode.capitalize()) + except: + path = make_timestamp_path() + + return save_history(history, path) + + def save_persistent_history(history, character, mode): - if mode in ['chat', 'chat-instruct'] and character not in ['', 'None', None] and not shared.args.multi_user: + if mode in ['chat', 'chat-instruct'] and character not in ['', 'None', None] and not shared.args.multi_user: save_history(history, path=Path(f'logs/{character}_persistent.json')) diff --git a/server.py b/server.py index 489936e5..edd91bc7 100644 --- a/server.py +++ b/server.py @@ -982,7 +982,7 @@ def create_interface(): lambda: 'characters/instruction-following/', None, gradio('delete_root')).then( lambda: gr.update(visible=True), None, gradio('file_deleter')) - shared.gradio['download_button'].click(chat.save_history, gradio('history'), gradio('download')) + shared.gradio['download_button'].click(chat.save_history_at_user_request, gradio('history', 'character_menu', 'mode'), gradio('download')) shared.gradio['Submit character'].click(chat.upload_character, gradio('upload_json', 'upload_img_bot'), gradio('character_menu')) shared.gradio['upload_json'].upload(lambda: gr.update(interactive=True), None, gradio('Submit character')) shared.gradio['upload_json'].clear(lambda: gr.update(interactive=False), None, gradio('Submit character')) From ad07839a7b8c8bf852e16e9e26d206351ffcc0ab Mon Sep 17 00:00:00 2001 From: original-subliminal-thought-criminal Date: Tue, 11 Jul 2023 22:16:36 -0500 Subject: [PATCH 36/54] Small bug, when arbitrary loading character.json that doesn't exist (#2643) * Fixes #2482 * corrected erroronius variable * Use .exists() --------- Co-authored-by: oobabooga <112222186+oobabooga@users.noreply.github.com> --- modules/chat.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/chat.py b/modules/chat.py index c0635c23..d1f85d62 100644 --- a/modules/chat.py +++ b/modules/chat.py @@ -460,11 +460,16 @@ def load_character(character, name1, name2, instruct=False): if character not in ['None', '', None]: folder = 'characters' if not instruct else 'characters/instruction-following' picture = generate_pfp_cache(character) + filepath = None for extension in ["yml", "yaml", "json"]: filepath = Path(f'{folder}/{character}.{extension}') if filepath.exists(): break + if filepath is None: + logger.error(f"Could not find character file for {character} in {folder} folder. Please check your spelling.") + return name1, name2, picture, greeting, context, turn_template.replace("\n", r"\n") + file_contents = open(filepath, 'r', encoding='utf-8').read() data = json.loads(file_contents) if extension == "json" else yaml.safe_load(file_contents) From 180420d2c9691eefcd8e60217b2fdb9efac1359b Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 11 Jul 2023 20:56:01 -0700 Subject: [PATCH 37/54] Fix send_pictures extension --- extensions/send_pictures/script.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/extensions/send_pictures/script.py b/extensions/send_pictures/script.py index dbbeb0fd..63421743 100644 --- a/extensions/send_pictures/script.py +++ b/extensions/send_pictures/script.py @@ -7,6 +7,7 @@ from transformers import BlipForConditionalGeneration, BlipProcessor from modules import chat, shared from modules.ui import gather_interface_values +from modules.utils import gradio # If 'state' is True, will hijack the next chat generation with # custom input text given by 'value' in the format [text, visible_text] @@ -42,6 +43,6 @@ def ui(): # Prepare the input hijack, update the interface values, call the generation function, and clear the picture picture_select.upload( lambda picture, name1, name2: input_hijack.update({"state": True, "value": generate_chat_picture(picture, name1, name2)}), [picture_select, shared.gradio['name1'], shared.gradio['name2']], None).then( - gather_interface_values, [shared.gradio[k] for k in shared.input_elements], shared.gradio['interface_state']).then( - chat.generate_chat_reply_wrapper, shared.input_params, shared.gradio['display'], show_progress=False).then( + gather_interface_values, gradio(shared.input_elements), gradio('interface_state')).then( + chat.generate_chat_reply_wrapper, shared.input_params, gradio('display', 'history'), show_progress=False).then( lambda: None, None, picture_select, show_progress=False) From eedb3bf0233392eae413f9a020c1bb6921be2af4 Mon Sep 17 00:00:00 2001 From: Gabriel Pena Date: Wed, 12 Jul 2023 11:05:13 -0300 Subject: [PATCH 38/54] Add low vram mode on llama cpp (#3076) --- modules/llamacpp_model.py | 1 + modules/loaders.py | 1 + modules/shared.py | 1 + modules/ui.py | 1 + server.py | 1 + 5 files changed, 5 insertions(+) diff --git a/modules/llamacpp_model.py b/modules/llamacpp_model.py index 4899ad99..86537a27 100644 --- a/modules/llamacpp_model.py +++ b/modules/llamacpp_model.py @@ -49,6 +49,7 @@ class LlamaCppModel: 'n_batch': shared.args.n_batch, 'use_mmap': not shared.args.no_mmap, 'use_mlock': shared.args.mlock, + 'low_vram': shared.args.low_vram, 'n_gpu_layers': shared.args.n_gpu_layers } diff --git a/modules/loaders.py b/modules/loaders.py index 8ec575a7..e0db482c 100644 --- a/modules/loaders.py +++ b/modules/loaders.py @@ -34,6 +34,7 @@ loaders_and_params = { 'n_batch', 'threads', 'no_mmap', + 'low_vram', 'mlock', 'llama_cpp_seed', ], diff --git a/modules/shared.py b/modules/shared.py index 2b2fa061..4b6b9fe1 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -120,6 +120,7 @@ parser.add_argument('--use_double_quant', action='store_true', help='use_double_ parser.add_argument('--threads', type=int, default=0, help='Number of threads to use.') parser.add_argument('--n_batch', type=int, default=512, help='Maximum number of prompt tokens to batch together when calling llama_eval.') parser.add_argument('--no-mmap', action='store_true', help='Prevent mmap from being used.') +parser.add_argument('--low-vram', action='store_true', help='Low VRAM Mode') parser.add_argument('--mlock', action='store_true', help='Force the system to keep the model in RAM.') parser.add_argument('--cache-capacity', type=str, help='Maximum cache capacity. Examples: 2000MiB, 2GiB. When provided without units, bytes will be assumed.') parser.add_argument('--n-gpu-layers', type=int, default=0, help='Number of layers to offload to the GPU.') diff --git a/modules/ui.py b/modules/ui.py index 9fea2880..704be925 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -57,6 +57,7 @@ def list_model_elements(): 'threads', 'n_batch', 'no_mmap', + 'low_vram', 'mlock', 'n_gpu_layers', 'n_ctx', diff --git a/server.py b/server.py index edd91bc7..2723b284 100644 --- a/server.py +++ b/server.py @@ -248,6 +248,7 @@ def create_model_menus(): shared.gradio['load_in_4bit'] = gr.Checkbox(label="load-in-4bit", value=shared.args.load_in_4bit) shared.gradio['use_double_quant'] = gr.Checkbox(label="use_double_quant", value=shared.args.use_double_quant) shared.gradio['no_mmap'] = gr.Checkbox(label="no-mmap", value=shared.args.no_mmap) + shared.gradio['low_vram'] = gr.Checkbox(label="low-vram", value=shared.args.low_vram) shared.gradio['mlock'] = gr.Checkbox(label="mlock", value=shared.args.mlock) shared.gradio['llama_cpp_seed'] = gr.Number(label='Seed (0 for random)', value=shared.args.llama_cpp_seed) shared.gradio['trust_remote_code'] = gr.Checkbox(label="trust-remote-code", value=shared.args.trust_remote_code, info='Make sure to inspect the .py files inside the model folder before loading it with this option enabled.') From a17b78d334bfe3dc8eafc0243af895fa1ef728bf Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Wed, 12 Jul 2023 07:19:12 -0700 Subject: [PATCH 39/54] Disable wandb during training --- modules/training.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/modules/training.py b/modules/training.py index 29383e7f..fa9281bb 100644 --- a/modules/training.py +++ b/modules/training.py @@ -1,3 +1,8 @@ +import os + +os.environ["WANDB_MODE"] = "offline" +os.environ["WANDB_DISABLED"] = "true" + import json import math import random @@ -517,6 +522,7 @@ def do_train(lora_name: str, always_override: bool, save_steps: int, micro_batch train_dataset=train_data, eval_dataset=eval_data, args=transformers.TrainingArguments( + report_to=None, per_device_train_batch_size=micro_batch_size, gradient_accumulation_steps=gradient_accumulation_steps, warmup_steps=math.ceil(warmup_steps / gradient_accumulation_steps), From 73a0def4af8e05aff3af05732f5d1e787cbd8616 Mon Sep 17 00:00:00 2001 From: practicaldreamer <78515588+practicaldreamer@users.noreply.github.com> Date: Wed, 12 Jul 2023 09:26:45 -0500 Subject: [PATCH 40/54] Add Feature to Log Sample of Training Dataset for Inspection (#1711) --- modules/training.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/modules/training.py b/modules/training.py index fa9281bb..442b92b3 100644 --- a/modules/training.py +++ b/modules/training.py @@ -579,8 +579,27 @@ def do_train(lora_name: str, always_override: bool, save_steps: int, micro_batch if WANT_INTERRUPT: yield "Interrupted before start." return + + def log_train_dataset(trainer): + decoded_entries = [] + # Try to decode the entries and write the log file + try: + # Iterate over the first 10 elements in the dataset (or fewer if there are less than 10) + for i in range(min(10, len(trainer.train_dataset))): + decoded_text = shared.tokenizer.decode(trainer.train_dataset[i]['input_ids']) + decoded_entries.append({"value": decoded_text}) + + # Write the log file + Path('logs').mkdir(exist_ok=True) + with open(Path('logs/train_dataset_sample.json'), 'w') as json_file: + json.dump(decoded_entries, json_file, indent=4) + + logger.info("Log file 'train_dataset_sample.json' created in the 'logs' directory.") + except Exception as e: + logger.error(f"Failed to create log file due to error: {e}") def threaded_run(): + log_train_dataset(trainer) trainer.train() # Note: save in the thread in case the gradio thread breaks (eg browser closed) lora_model.save_pretrained(lora_file_path) From 5d513eea222f29f1e1c1b0a57500134873bf7c0f Mon Sep 17 00:00:00 2001 From: kizinfo Date: Wed, 12 Jul 2023 17:44:30 +0300 Subject: [PATCH 41/54] Add ability to load all text files from a subdirectory for training (#1997) * Update utils.py returns individual txt files and subdirectories to getdatasets to allow for training from a directory of text files * Update training.py minor tweak to training on raw datasets to detect if a directory is selected, and if so, to load in all the txt files in that directory for training * Update put-trainer-datasets-here.txt document * Minor change * Use pathlib, sort by natural keys * Space --------- Co-authored-by: oobabooga <112222186+oobabooga@users.noreply.github.com> --- modules/training.py | 22 ++++++++++++++----- modules/utils.py | 4 ++++ .../datasets/put-trainer-datasets-here.txt | 1 + 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/modules/training.py b/modules/training.py index 442b92b3..2f9a7768 100644 --- a/modules/training.py +++ b/modules/training.py @@ -32,6 +32,7 @@ from modules.evaluate import ( save_past_evaluations ) from modules.logging_colors import logger +from modules.utils import natural_keys # This mapping is from a very recent commit, not yet released. # If not available, default to a backup map for some common model types. @@ -354,12 +355,23 @@ def do_train(lora_name: str, always_override: bool, save_steps: int, micro_batch # == Prep the dataset, format, etc == if raw_text_file not in ['None', '']: - logger.info("Loading raw text file dataset...") - train_template["template_type"] = "raw_text" + logger.info("Loading raw text file dataset...") + fullpath = clean_path('training/datasets', f'{raw_text_file}') + fullpath = Path(fullpath) + if fullpath.is_dir(): + logger.info('Training path directory {}'.format(raw_text_file)) + raw_text = "" + file_paths = sorted(fullpath.glob('*.txt'), key=lambda path: natural_keys(path.name)) + for file_path in file_paths: + if file_path.is_file(): + with file_path.open('r', encoding='utf-8') as file: + raw_text += file.read() - with open(clean_path('training/datasets', f'{raw_text_file}.txt'), 'r', encoding='utf-8') as file: - raw_text = file.read().replace('\r', '') + logger.info(f"Loaded training file: {file_path.name}") + else: + with open(clean_path('training/datasets', f'{raw_text_file}.txt'), 'r', encoding='utf-8') as file: + raw_text = file.read() cut_string = hard_cut_string.replace('\\n', '\n') out_tokens = [] @@ -579,7 +591,7 @@ def do_train(lora_name: str, always_override: bool, save_steps: int, micro_batch if WANT_INTERRUPT: yield "Interrupted before start." return - + def log_train_dataset(trainer): decoded_entries = [] # Try to decode the entries and write the log file diff --git a/modules/utils.py b/modules/utils.py index 72a0dfa1..8b662be1 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -114,6 +114,10 @@ def get_available_loras(): def get_datasets(path: str, ext: str): + # include subdirectories for raw txt files to allow training from a subdirectory of txt files + if ext == "txt": + return ['None'] + sorted(set([k.stem for k in list(Path(path).glob('txt'))+list(Path(path).glob('*/')) if k.stem != 'put-trainer-datasets-here']), key=natural_keys) + return ['None'] + sorted(set([k.stem for k in Path(path).glob(f'*.{ext}') if k.stem != 'put-trainer-datasets-here']), key=natural_keys) diff --git a/training/datasets/put-trainer-datasets-here.txt b/training/datasets/put-trainer-datasets-here.txt index e69de29b..932eacf8 100644 --- a/training/datasets/put-trainer-datasets-here.txt +++ b/training/datasets/put-trainer-datasets-here.txt @@ -0,0 +1 @@ +to load multiple raw text files create a subdirectory and put them all there From 3f19e94c9377489fb4f8716864f01ea825e2ffad Mon Sep 17 00:00:00 2001 From: kabachuha Date: Wed, 12 Jul 2023 17:53:31 +0300 Subject: [PATCH 42/54] Add Tensorboard/Weights and biases integration for training (#2624) --- modules/training.py | 15 +++++++++------ requirements.txt | 2 ++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/modules/training.py b/modules/training.py index 2f9a7768..22dce3fc 100644 --- a/modules/training.py +++ b/modules/training.py @@ -1,7 +1,7 @@ import os os.environ["WANDB_MODE"] = "offline" -os.environ["WANDB_DISABLED"] = "true" +# os.environ["WANDB_DISABLED"] = "true" import json import math @@ -60,7 +60,7 @@ train_log = {} train_template = {} WANT_INTERRUPT = False -PARAMETERS = ["lora_name", "always_override", "save_steps", "micro_batch_size", "batch_size", "epochs", "learning_rate", "lr_scheduler_type", "lora_rank", "lora_alpha", "lora_dropout", "cutoff_len", "dataset", "eval_dataset", "format", "eval_steps", "raw_text_file", "overlap_len", "newline_favor_len", "higher_rank_limit", "warmup_steps", "optimizer", "hard_cut_string", "train_only_after", "stop_at_loss"] +PARAMETERS = ["lora_name", "always_override", "save_steps", "micro_batch_size", "batch_size", "epochs", "learning_rate", "lr_scheduler_type", "lora_rank", "lora_alpha", "lora_dropout", "cutoff_len", "dataset", "eval_dataset", "format", "eval_steps", "raw_text_file", "overlap_len", "newline_favor_len", "higher_rank_limit", "warmup_steps", "optimizer", "hard_cut_string", "train_only_after", "stop_at_loss", "report_to"] def create_train_interface(): @@ -122,6 +122,8 @@ def create_train_interface(): with gr.Row(): higher_rank_limit = gr.Checkbox(label='Enable higher ranks', value=False, info='If checked, changes Rank/Alpha slider above to go much higher. This will not work without a datacenter-class GPU.') + with gr.Row(): + report_to = gr.Radio(label="Save detailed logs with", value="None", choices=["None", "wandb", "tensorboard"], interactive=True) with gr.Row(): start_button = gr.Button("Start LoRA Training") @@ -152,7 +154,8 @@ def create_train_interface(): refresh_table = gr.Button('Refresh the table', elem_classes="small-button") # Training events - all_params = [lora_name, always_override, save_steps, micro_batch_size, batch_size, epochs, learning_rate, lr_scheduler_type, lora_rank, lora_alpha, lora_dropout, cutoff_len, dataset, eval_dataset, format, eval_steps, raw_text_file, overlap_len, newline_favor_len, higher_rank_limit, warmup_steps, optimizer, hard_cut_string, train_only_after, stop_at_loss] + all_params = [lora_name, always_override, save_steps, micro_batch_size, batch_size, epochs, learning_rate, lr_scheduler_type, lora_rank, lora_alpha, lora_dropout, cutoff_len, dataset, eval_dataset, format, eval_steps, raw_text_file, overlap_len, newline_favor_len, higher_rank_limit, warmup_steps, optimizer, hard_cut_string, train_only_after, stop_at_loss, report_to] + copy_from.change(do_copy_params, [copy_from] + all_params, all_params) start_button.click(do_train, all_params, output) stop_button.click(do_interrupt, None, None, queue=False) @@ -261,7 +264,7 @@ def calc_trainable_parameters(model): return trainable_params, all_param -def do_train(lora_name: str, always_override: bool, save_steps: int, micro_batch_size: int, batch_size: int, epochs: int, learning_rate: str, lr_scheduler_type: str, lora_rank: int, lora_alpha: int, lora_dropout: float, cutoff_len: int, dataset: str, eval_dataset: str, format: str, eval_steps: int, raw_text_file: str, overlap_len: int, newline_favor_len: int, higher_rank_limit: bool, warmup_steps: int, optimizer: str, hard_cut_string: str, train_only_after: str, stop_at_loss: float): +def do_train(lora_name: str, always_override: bool, save_steps: int, micro_batch_size: int, batch_size: int, epochs: int, learning_rate: str, lr_scheduler_type: str, lora_rank: int, lora_alpha: int, lora_dropout: float, cutoff_len: int, dataset: str, eval_dataset: str, format: str, eval_steps: int, raw_text_file: str, overlap_len: int, newline_favor_len: int, higher_rank_limit: bool, warmup_steps: int, optimizer: str, hard_cut_string: str, train_only_after: str, stop_at_loss: float, report_to: str): if shared.args.monkey_patch: from monkeypatch.peft_tuners_lora_monkey_patch import ( @@ -534,7 +537,7 @@ def do_train(lora_name: str, always_override: bool, save_steps: int, micro_batch train_dataset=train_data, eval_dataset=eval_data, args=transformers.TrainingArguments( - report_to=None, + report_to=report_to if report_to != "None" else None, per_device_train_batch_size=micro_batch_size, gradient_accumulation_steps=gradient_accumulation_steps, warmup_steps=math.ceil(warmup_steps / gradient_accumulation_steps), @@ -551,7 +554,7 @@ def do_train(lora_name: str, always_override: bool, save_steps: int, micro_batch load_best_model_at_end=eval_data is not None, # TODO: Enable multi-device support ddp_find_unused_parameters=None, - no_cuda=shared.args.cpu + no_cuda=shared.args.cpu, ), data_collator=transformers.DataCollatorForLanguageModeling(shared.tokenizer, mlm=False), callbacks=list([Callbacks()]) diff --git a/requirements.txt b/requirements.txt index 681be6d4..4e012c1e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,6 +16,8 @@ safetensors==0.3.1 sentencepiece tqdm scipy +tensorboard +wandb transformers==4.30.2 git+https://github.com/huggingface/peft@03eb378eb914fbee709ff7c86ba5b1d033b89524 bitsandbytes==0.40.0; platform_system != "Windows" From 987d0fe023d4a536193a54dbfc5ef6cf8b54859a Mon Sep 17 00:00:00 2001 From: Fernando Tarin Morales Date: Thu, 13 Jul 2023 00:05:37 +0900 Subject: [PATCH 43/54] Fix: Fixed the tokenization process of a raw dataset and improved its efficiency (#3035) --- modules/training.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/modules/training.py b/modules/training.py index 22dce3fc..9388436b 100644 --- a/modules/training.py +++ b/modules/training.py @@ -388,12 +388,7 @@ def do_train(lora_name: str, always_override: bool, save_steps: int, micro_batch yield f"Error: overlap_len ({overlap_len}) cannot be greater than or equal to cutoff_len ({cutoff_len})" return - tokens = list(split_chunks(tokens, step)) - for i in range(1, len(tokens)): - tokens[i] = tokens[i - 1][-overlap_len:] + tokens[i] - - out_tokens.extend(tokens) - del tokens + out_tokens.extend(split_chunks(tokens, cutoff_len, step)) del raw_text # Note: could be a gig for a large dataset, so delete redundant data as we go to be safe on RAM text_chunks = [shared.tokenizer.decode(x) for x in out_tokens] @@ -663,9 +658,9 @@ def do_train(lora_name: str, always_override: bool, save_steps: int, micro_batch yield f"Done! LoRA saved to `{lora_file_path}`" -def split_chunks(arr, step): +def split_chunks(arr, size, step): for i in range(0, len(arr), step): - yield arr[i:i + step] + yield arr[i:i + size] def cut_chunk_for_newline(chunk: str, max_length: int): From 30f37530d506aad0fd41c87e1ec0aa8aaf897e71 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Wed, 12 Jul 2023 09:52:20 -0700 Subject: [PATCH 44/54] Add back .replace('\r', '') --- modules/training.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/training.py b/modules/training.py index 9388436b..11c36611 100644 --- a/modules/training.py +++ b/modules/training.py @@ -369,12 +369,12 @@ def do_train(lora_name: str, always_override: bool, save_steps: int, micro_batch for file_path in file_paths: if file_path.is_file(): with file_path.open('r', encoding='utf-8') as file: - raw_text += file.read() + raw_text += file.read().replace('\r', '') logger.info(f"Loaded training file: {file_path.name}") else: with open(clean_path('training/datasets', f'{raw_text_file}.txt'), 'r', encoding='utf-8') as file: - raw_text = file.read() + raw_text = file.read().replace('\r', '') cut_string = hard_cut_string.replace('\\n', '\n') out_tokens = [] From 9b55d3a9f9d24af66566aae8ad3881625f6432b4 Mon Sep 17 00:00:00 2001 From: FartyPants Date: Wed, 12 Jul 2023 14:29:43 -0400 Subject: [PATCH 45/54] More robust and error prone training (#3058) --- modules/chat.py | 5 ++++ modules/models.py | 1 + modules/shared.py | 1 + modules/training.py | 64 ++++++++++++++++++++++++++++++++++++++++----- 4 files changed, 65 insertions(+), 6 deletions(-) diff --git a/modules/chat.py b/modules/chat.py index be524ad0..d2423555 100644 --- a/modules/chat.py +++ b/modules/chat.py @@ -478,11 +478,16 @@ def load_character(character, name1, name2, instruct=False): if character not in ['None', '', None]: folder = 'characters' if not instruct else 'characters/instruction-following' picture = generate_pfp_cache(character) + filepath = None for extension in ["yml", "yaml", "json"]: filepath = Path(f'{folder}/{character}.{extension}') if filepath.exists(): break + if filepath is None: + logger.error(f"Could not find character file for {character} in {folder} folder. Please check your spelling.") + return name1, name2, picture, greeting, context, turn_template.replace("\n", r"\n") + file_contents = open(filepath, 'r', encoding='utf-8').read() data = json.loads(file_contents) if extension == "json" else yaml.safe_load(file_contents) diff --git a/modules/models.py b/modules/models.py index cd01899e..497d6b78 100644 --- a/modules/models.py +++ b/modules/models.py @@ -339,6 +339,7 @@ def clear_torch_cache(): def unload_model(): shared.model = shared.tokenizer = None shared.lora_names = [] + shared.model_dirty_from_training = False clear_torch_cache() diff --git a/modules/shared.py b/modules/shared.py index 4b6b9fe1..14a10e57 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -12,6 +12,7 @@ tokenizer = None is_seq2seq = False model_name = "None" lora_names = [] +model_dirty_from_training = False # Chat variables stop_everything = False diff --git a/modules/training.py b/modules/training.py index 11c36611..1f8e5e5e 100644 --- a/modules/training.py +++ b/modules/training.py @@ -17,6 +17,8 @@ from pathlib import Path import gradio as gr import torch import transformers +from modules.models import load_model, unload_model + from datasets import Dataset, load_dataset from peft import ( LoraConfig, @@ -60,7 +62,7 @@ train_log = {} train_template = {} WANT_INTERRUPT = False -PARAMETERS = ["lora_name", "always_override", "save_steps", "micro_batch_size", "batch_size", "epochs", "learning_rate", "lr_scheduler_type", "lora_rank", "lora_alpha", "lora_dropout", "cutoff_len", "dataset", "eval_dataset", "format", "eval_steps", "raw_text_file", "overlap_len", "newline_favor_len", "higher_rank_limit", "warmup_steps", "optimizer", "hard_cut_string", "train_only_after", "stop_at_loss", "report_to"] +PARAMETERS = ["lora_name", "always_override", "save_steps", "micro_batch_size", "batch_size", "epochs", "learning_rate", "lr_scheduler_type", "lora_rank", "lora_alpha", "lora_dropout", "cutoff_len", "dataset", "eval_dataset", "format", "eval_steps", "raw_text_file", "overlap_len", "newline_favor_len", "higher_rank_limit", "warmup_steps", "optimizer", "hard_cut_string", "train_only_after", "stop_at_loss", "add_eos_token", "min_chars", "report_to"] def create_train_interface(): @@ -108,6 +110,7 @@ def create_train_interface(): raw_text_file = gr.Dropdown(choices=utils.get_datasets('training/datasets', 'txt'), value='None', label='Text file', info='The raw text file to use for training.') ui.create_refresh_button(raw_text_file, lambda: None, lambda: {'choices': utils.get_datasets('training/datasets', 'txt')}, 'refresh-button') hard_cut_string = gr.Textbox(label='Hard Cut String', value='\\n\\n\\n', info='String that indicates a hard cut between text parts. Helps prevent unwanted overlap.') + min_chars = gr.Number(label='Ignore small blocks', value=0, info='Ignore Hard Cut blocks that have less or equal characters than this number') with gr.Row(): overlap_len = gr.Slider(label='Overlap Length', minimum=0, maximum=512, value=128, step=16, info='Overlap length - ie how many tokens from the prior chunk of text to include into the next chunk. (The chunks themselves will be of a size determined by Cutoff Length below). Setting overlap to exactly half the cutoff length may be ideal.') @@ -119,6 +122,7 @@ def create_train_interface(): optimizer = gr.Dropdown(label='Optimizer', value='adamw_torch', choices=['adamw_hf', 'adamw_torch', 'adamw_torch_fused', 'adamw_torch_xla', 'adamw_apex_fused', 'adafactor', 'adamw_bnb_8bit', 'adamw_anyprecision', 'sgd', 'adagrad'], info='Different optimizer implementation options, for advanced users. Effects of different options are not well documented yet.') train_only_after = gr.Textbox(label='Train Only After', value='', info='Only consider text *after* this string in any given chunk for training. For Alpaca datasets, use "### Response:" to only train the response and ignore the input.') stop_at_loss = gr.Slider(label='Stop at loss', minimum=0.0, maximum=3.0, step=0.1, value=0.00, info='The process will automatically stop once the desired loss value is reached. (reasonable numbers are 1.5-1.8)') + add_eos_token = gr.Checkbox(label='Add EOS token', value=False, info="Adds EOS token for each dataset item. In case of raw text, the EOS will be added at the Hard Cut") with gr.Row(): higher_rank_limit = gr.Checkbox(label='Enable higher ranks', value=False, info='If checked, changes Rank/Alpha slider above to go much higher. This will not work without a datacenter-class GPU.') @@ -154,7 +158,8 @@ def create_train_interface(): refresh_table = gr.Button('Refresh the table', elem_classes="small-button") # Training events - all_params = [lora_name, always_override, save_steps, micro_batch_size, batch_size, epochs, learning_rate, lr_scheduler_type, lora_rank, lora_alpha, lora_dropout, cutoff_len, dataset, eval_dataset, format, eval_steps, raw_text_file, overlap_len, newline_favor_len, higher_rank_limit, warmup_steps, optimizer, hard_cut_string, train_only_after, stop_at_loss, report_to] + + all_params = [lora_name, always_override, save_steps, micro_batch_size, batch_size, epochs, learning_rate, lr_scheduler_type, lora_rank, lora_alpha, lora_dropout, cutoff_len, dataset, eval_dataset, format, eval_steps, raw_text_file, overlap_len, newline_favor_len, higher_rank_limit, warmup_steps, optimizer, hard_cut_string, train_only_after, stop_at_loss, add_eos_token, min_chars, report_to] copy_from.change(do_copy_params, [copy_from] + all_params, all_params) start_button.click(do_train, all_params, output) @@ -264,7 +269,7 @@ def calc_trainable_parameters(model): return trainable_params, all_param -def do_train(lora_name: str, always_override: bool, save_steps: int, micro_batch_size: int, batch_size: int, epochs: int, learning_rate: str, lr_scheduler_type: str, lora_rank: int, lora_alpha: int, lora_dropout: float, cutoff_len: int, dataset: str, eval_dataset: str, format: str, eval_steps: int, raw_text_file: str, overlap_len: int, newline_favor_len: int, higher_rank_limit: bool, warmup_steps: int, optimizer: str, hard_cut_string: str, train_only_after: str, stop_at_loss: float, report_to: str): +def do_train(lora_name: str, always_override: bool, save_steps: int, micro_batch_size: int, batch_size: int, epochs: int, learning_rate: str, lr_scheduler_type: str, lora_rank: int, lora_alpha: int, lora_dropout: float, cutoff_len: int, dataset: str, eval_dataset: str, format: str, eval_steps: int, raw_text_file: str, overlap_len: int, newline_favor_len: int, higher_rank_limit: bool, warmup_steps: int, optimizer: str, hard_cut_string: str, train_only_after: str, stop_at_loss: float, add_eos_token: bool, min_chars: int, report_to: str): if shared.args.monkey_patch: from monkeypatch.peft_tuners_lora_monkey_patch import ( @@ -322,14 +327,22 @@ def do_train(lora_name: str, always_override: bool, save_steps: int, micro_batch def encode(text, add_bos_token): result = shared.tokenizer.encode(text, truncation=True, max_length=cutoff_len) + # Check if the first two tokens are BOS + if len(result) >= 2 and result[:2] == [shared.tokenizer.bos_token_id, shared.tokenizer.bos_token_id]: + result = result[1:] + if not add_bos_token and result[0] == shared.tokenizer.bos_token_id: result = result[1:] return result - def tokenize(prompt): + def tokenize(prompt, append_eos_token=False): if train_only_after == '' or train_only_after not in prompt: input_ids = encode(prompt, True) + + if append_eos_token and input_ids[-1] != shared.tokenizer.eos_token_id and len(input_ids) < cutoff_len: + input_ids.append(shared.tokenizer.eos_token_id) + input_ids = [shared.tokenizer.pad_token_id] * (cutoff_len - len(input_ids)) + input_ids labels = [1] * len(input_ids) @@ -338,6 +351,9 @@ def do_train(lora_name: str, always_override: bool, save_steps: int, micro_batch before_tokens = encode(prompt[:ind], True) after_tokens = encode(prompt[ind:], False) + if append_eos_token and after_tokens[-1] != shared.tokenizer.eos_token_id: + after_tokens.append(shared.tokenizer.eos_token_id) + full_length = len(after_tokens) + len(before_tokens) if full_length > cutoff_len: after_tokens = after_tokens[:cutoff_len - len(before_tokens)] @@ -377,12 +393,18 @@ def do_train(lora_name: str, always_override: bool, save_steps: int, micro_batch raw_text = file.read().replace('\r', '') cut_string = hard_cut_string.replace('\\n', '\n') + eos_added = 0 out_tokens = [] for text_part in raw_text.split(cut_string): - if text_part.strip() == '': + + if len(text_part.strip()) <= min_chars: continue tokens = shared.tokenizer.encode(text_part) + if add_eos_token: + tokens.append(shared.tokenizer.eos_token_id) + eos_added += 1 + step = cutoff_len - overlap_len if step <= 0: yield f"Error: overlap_len ({overlap_len}) cannot be greater than or equal to cutoff_len ({cutoff_len})" @@ -390,6 +412,9 @@ def do_train(lora_name: str, always_override: bool, save_steps: int, micro_batch out_tokens.extend(split_chunks(tokens, cutoff_len, step)) + if eos_added > 0: + print(f"EOS added to {eos_added} text blocks") + del raw_text # Note: could be a gig for a large dataset, so delete redundant data as we go to be safe on RAM text_chunks = [shared.tokenizer.decode(x) for x in out_tokens] del out_tokens @@ -429,7 +454,7 @@ def do_train(lora_name: str, always_override: bool, save_steps: int, micro_batch def generate_and_tokenize_prompt(data_point): prompt = generate_prompt(data_point) - return tokenize(prompt) + return tokenize(prompt, add_eos_token) logger.info("Loading JSON datasets...") data = load_dataset("json", data_files=clean_path('training/datasets', f'{dataset}.json')) @@ -441,11 +466,33 @@ def do_train(lora_name: str, always_override: bool, save_steps: int, micro_batch eval_data = load_dataset("json", data_files=clean_path('training/datasets', f'{eval_dataset}.json')) eval_data = eval_data['train'].map(generate_and_tokenize_prompt, new_fingerprint='%030x' % random.randrange(16**30)) + # == We MUST reload model if it went through any previous training, even failed one == + if shared.model_dirty_from_training: + selected_model = shared.model_name + if selected_model: + print("\033[1;31;1m(Model has been modified by previous training, it needs to be reloaded...)\033[0;37;0m") + try: + yield f"Reloading {selected_model}..." + unload_model() + shared.model, shared.tokenizer = load_model(shared.model_name, None) + if shared.model is not None: + print("Model reloaded OK, continue with training.") + else: + return f"Failed to load {selected_model}." + except: + exc = traceback.format_exc() + logger.error('Failed to reload the model.') + print(exc) + return exc + # == Start prepping the model itself == if not hasattr(shared.model, 'lm_head') or hasattr(shared.model.lm_head, 'weight'): logger.info("Getting model ready...") prepare_model_for_int8_training(shared.model) + # base model is now frozen and should not be reused for any other LoRA training than this one + shared.model_dirty_from_training = True + logger.info("Prepping for training...") config = LoraConfig( r=lora_rank, @@ -575,6 +622,10 @@ def do_train(lora_name: str, always_override: bool, save_steps: int, micro_batch lora_trainable_param, lora_all_param = calc_trainable_parameters(lora_model) + projections_string = ", ".join([projection.replace("_proj", "") for projection in model_to_lora_modules[model_id]]) + + print(f"Training '{model_id}' model using ({projections_string}) projections") + if lora_all_param > 0: print(f"Trainable params: {lora_trainable_param:,d} ({100 * lora_trainable_param / lora_all_param:.4f} %), All params: {lora_all_param:,d} (Model: {model_all_params:,d})") @@ -582,6 +633,7 @@ def do_train(lora_name: str, always_override: bool, save_steps: int, micro_batch train_log.update({"base_model_class": shared.model.__class__.__name__}) train_log.update({"base_loaded_in_4bit": getattr(lora_model, "is_loaded_in_4bit", False)}) train_log.update({"base_loaded_in_8bit": getattr(lora_model, "is_loaded_in_8bit", False)}) + train_log.update({"projections": projections_string}) if stop_at_loss > 0: print(f"Monitoring loss \033[1;31;1m(Auto-Stop at: {stop_at_loss})\033[0;37;0m") From e202190c4f739a7fd62ee3f0034064635fe7ca20 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Wed, 12 Jul 2023 11:33:25 -0700 Subject: [PATCH 46/54] lint --- download-model.py | 2 +- extensions/api/util.py | 2 +- extensions/elevenlabs_tts/script.py | 2 +- extensions/ngrok/script.py | 6 +- extensions/openai/completions.py | 92 ++++++++++++++-------------- extensions/openai/defaults.py | 11 ++-- extensions/openai/edits.py | 14 ++--- extensions/openai/embeddings.py | 12 ++-- extensions/openai/errors.py | 10 ++- extensions/openai/images.py | 9 +-- extensions/openai/models.py | 18 +++--- extensions/openai/moderations.py | 15 +++-- extensions/openai/script.py | 23 +++---- extensions/openai/tokens.py | 11 ++-- extensions/openai/utils.py | 5 +- extensions/sd_api_pictures/script.py | 4 ++ extensions/superbooga/script.py | 2 +- extensions/whisper_stt/script.py | 12 ++-- modules/LoRA.py | 8 +-- modules/loaders.py | 4 +- modules/models.py | 4 +- modules/sampler_hijack.py | 1 + modules/shared.py | 2 +- modules/utils.py | 2 +- 24 files changed, 146 insertions(+), 125 deletions(-) diff --git a/download-model.py b/download-model.py index 34986c75..f5d49064 100644 --- a/download-model.py +++ b/download-model.py @@ -23,7 +23,7 @@ from tqdm.contrib.concurrent import thread_map class ModelDownloader: - def __init__(self, max_retries = 5): + def __init__(self, max_retries=5): self.s = requests.Session() if max_retries: self.s.mount('https://cdn-lfs.huggingface.co', HTTPAdapter(max_retries=max_retries)) diff --git a/extensions/api/util.py b/extensions/api/util.py index a25c7885..a9d581eb 100644 --- a/extensions/api/util.py +++ b/extensions/api/util.py @@ -75,7 +75,7 @@ def build_parameters(body, chat=False): 'greeting': greeting, 'name1_instruct': name1_instruct, 'name2_instruct': name2_instruct, - 'context_instruct': body.get('context_instruct', context_instruct), + 'context_instruct': body.get('context_instruct', context_instruct), 'turn_template': turn_template, 'chat-instruct_command': str(body.get('chat-instruct_command', shared.settings['chat-instruct_command'])), 'history': body.get('history', {'internal': [], 'visible': []}) diff --git a/extensions/elevenlabs_tts/script.py b/extensions/elevenlabs_tts/script.py index a64822ce..f74e1047 100644 --- a/extensions/elevenlabs_tts/script.py +++ b/extensions/elevenlabs_tts/script.py @@ -160,7 +160,7 @@ def ui(): api_key = gr.Textbox(placeholder="Enter your API key.", label='API Key') with gr.Row(): - model = gr.Dropdown(value=params['model'], choices=LANG_MODELS, label='Language model') + model = gr.Dropdown(value=params['model'], choices=LANG_MODELS, label='Language model') with gr.Row(): convert = gr.Button('Permanently replace audios with the message texts') diff --git a/extensions/ngrok/script.py b/extensions/ngrok/script.py index 782deeac..46f39bd3 100644 --- a/extensions/ngrok/script.py +++ b/extensions/ngrok/script.py @@ -1,8 +1,8 @@ # Adds ngrok ingress, to use add `--extension ngrok` to the command line options # -# Parameters can be customized in settings.json of webui, e.g.: +# Parameters can be customized in settings.json of webui, e.g.: # {"ngrok": {"basic_auth":"user:password"} } -# or +# or # {"ngrok": {"oauth_provider":"google", "oauth_allow_emails":["asdf@asdf.com"]} } # # See this example for full list of options: https://github.com/ngrok/ngrok-py/blob/main/examples/ngrok-connect-full.py @@ -22,6 +22,7 @@ options = { 'session_metadata': 'text-generation-webui', } + def ui(): settings = shared.settings.get("ngrok") if settings: @@ -33,4 +34,3 @@ def ui(): logging.info(f"Ingress established at: {tunnel.url()}") except ModuleNotFoundError: logging.error("===> ngrok library not found, please run `pip install -r extensions/ngrok/requirements.txt`") - diff --git a/extensions/openai/completions.py b/extensions/openai/completions.py index ab1879b7..23c5dbee 100644 --- a/extensions/openai/completions.py +++ b/extensions/openai/completions.py @@ -36,12 +36,12 @@ class LogprobProcessor(LogitsProcessor): super().__init__() def __call__(self, input_ids: torch.LongTensor, logits: torch.FloatTensor) -> torch.FloatTensor: - if self.logprobs is not None: # 0-5 + if self.logprobs is not None: # 0-5 log_e_probabilities = F.log_softmax(logits, dim=1) # XXX hack. should find the selected token and include the prob of that # ... but we just +1 here instead because we don't know it yet. - top_values, top_indices = torch.topk(log_e_probabilities, k=self.logprobs + 1) - top_tokens = [ decode(tok) for tok in top_indices[0] ] + top_values, top_indices = torch.topk(log_e_probabilities, k=self.logprobs + 1) + top_tokens = [decode(tok) for tok in top_indices[0]] self.token_alternatives = dict(zip(top_tokens, top_values[0].tolist())) return logits @@ -50,9 +50,9 @@ def convert_logprobs_to_tiktoken(model, logprobs): try: encoder = tiktoken.encoding_for_model(model) # just pick the first one if it encodes to multiple tokens... 99.9% not required and maybe worse overall. - return dict([ (encoder.decode([encoder.encode(token)[0]]), prob) for token, prob in logprobs.items() ]) + return dict([(encoder.decode([encoder.encode(token)[0]]), prob) for token, prob in logprobs.items()]) except KeyError: - # assume native tokens if we can't find the tokenizer + # assume native tokens if we can't find the tokenizer return logprobs @@ -71,9 +71,9 @@ def marshal_common_params(body): # OpenAI API Parameters # model - ignored for now, TODO: When we can reliably load a model or lora from a name only change this req_params['requested_model'] = body.get('model', shared.model_name) - + req_params['suffix'] = default(body, 'suffix', req_params['suffix']) - req_params['temperature'] = clamp(default(body, 'temperature', req_params['temperature']), 0.001, 1.999) # fixup absolute 0.0/2.0 + req_params['temperature'] = clamp(default(body, 'temperature', req_params['temperature']), 0.001, 1.999) # fixup absolute 0.0/2.0 req_params['top_p'] = clamp(default(body, 'top_p', req_params['top_p']), 0.001, 1.0) n = default(body, 'n', 1) if n != 1: @@ -81,7 +81,7 @@ def marshal_common_params(body): if 'stop' in body: # str or array, max len 4 (ignored) if isinstance(body['stop'], str): - req_params['stopping_strings'] = [body['stop']] # non-standard parameter + req_params['stopping_strings'] = [body['stop']] # non-standard parameter elif isinstance(body['stop'], list): req_params['stopping_strings'] = body['stop'] @@ -91,7 +91,7 @@ def marshal_common_params(body): logits_processor = [] logit_bias = body.get('logit_bias', None) - if logit_bias: # {str: float, ...} + if logit_bias: # {str: float, ...} # XXX convert tokens from tiktoken based on requested model # Ex.: 'logit_bias': {'1129': 100, '11442': 100, '16243': 100} try: @@ -103,19 +103,19 @@ def marshal_common_params(body): print(logit_bias, '->', new_logit_bias) logit_bias = new_logit_bias except KeyError: - pass # assume native tokens if we can't find the tokenizer + pass # assume native tokens if we can't find the tokenizer logits_processor = [LogitsBiasProcessor(logit_bias)] - logprobs = None # coming to chat eventually + logprobs = None # coming to chat eventually if 'logprobs' in body: - logprobs = default(body, 'logprobs', 0) # maybe cap at topk? don't clamp 0-5. + logprobs = default(body, 'logprobs', 0) # maybe cap at topk? don't clamp 0-5. req_params['logprob_proc'] = LogprobProcessor(logprobs) logits_processor.extend([req_params['logprob_proc']]) else: logprobs = None - if logits_processor: # requires logits_processor support + if logits_processor: # requires logits_processor support req_params['logits_processor'] = LogitsProcessorList(logits_processor) return req_params @@ -123,14 +123,14 @@ def marshal_common_params(body): def messages_to_prompt(body: dict, req_params: dict, max_tokens): # functions - if body.get('functions', []): # chat only + if body.get('functions', []): # chat only raise InvalidRequestError(message="functions is not supported.", param='functions') - if body.get('function_call', ''): # chat only, 'none', 'auto', {'name': 'func'} + if body.get('function_call', ''): # chat only, 'none', 'auto', {'name': 'func'} raise InvalidRequestError(message="function_call is not supported.", param='function_call') if not 'messages' in body: raise InvalidRequestError(message="messages is required", param='messages') - + messages = body['messages'] role_formats = { @@ -152,11 +152,11 @@ def messages_to_prompt(body: dict, req_params: dict, max_tokens): template = instruct['turn_template'] system_message_template = "{message}" system_message_default = instruct['context'] - bot_start = template.find('<|bot|>') # So far, 100% of instruction templates have this token + bot_start = template.find('<|bot|>') # So far, 100% of instruction templates have this token user_message_template = template[:bot_start].replace('<|user-message|>', '{message}').replace('<|user|>', instruct['user']) bot_message_template = template[bot_start:].replace('<|bot-message|>', '{message}').replace('<|bot|>', instruct['bot']) bot_prompt = bot_message_template[:bot_message_template.find('{message}')].rstrip(' ') - + role_formats = { 'user': user_message_template, 'assistant': bot_message_template, @@ -167,7 +167,7 @@ def messages_to_prompt(body: dict, req_params: dict, max_tokens): if 'Alpaca' in shared.settings['instruction_template']: req_params['stopping_strings'].extend(['\n###']) - elif instruct['user']: # WizardLM and some others have no user prompt. + elif instruct['user']: # WizardLM and some others have no user prompt. req_params['stopping_strings'].extend(['\n' + instruct['user'], instruct['user']]) debug_msg(f"Loaded instruction role format: {shared.settings['instruction_template']}") @@ -220,16 +220,16 @@ def messages_to_prompt(body: dict, req_params: dict, max_tokens): if max_tokens > 0 and token_count + max_tokens > req_params['truncation_length']: err_msg = f"This model maximum context length is {req_params['truncation_length']} tokens. However, your messages resulted in over {token_count} tokens and max_tokens is {max_tokens}." print(f"Warning: ${err_msg}") - #raise InvalidRequestError(message=err_msg) + # raise InvalidRequestError(message=err_msg) return prompt, token_count -def chat_completions(body: dict, is_legacy: bool=False) -> dict: +def chat_completions(body: dict, is_legacy: bool = False) -> dict: # Chat Completions object_type = 'chat.completions' created_time = int(time.time()) - cmpl_id = "chatcmpl-%d" % (int(time.time()*1000000000)) + cmpl_id = "chatcmpl-%d" % (int(time.time() * 1000000000)) resp_list = 'data' if is_legacy else 'choices' # common params @@ -237,7 +237,7 @@ def chat_completions(body: dict, is_legacy: bool=False) -> dict: req_params['stream'] = False requested_model = req_params.pop('requested_model') logprob_proc = req_params.pop('logprob_proc', None) - req_params['top_k'] = 20 # There is no best_of/top_k param for chat, but it is much improved with a higher top_k. + req_params['top_k'] = 20 # There is no best_of/top_k param for chat, but it is much improved with a higher top_k. # chat default max_tokens is 'inf', but also flexible max_tokens = 0 @@ -254,7 +254,7 @@ def chat_completions(body: dict, is_legacy: bool=False) -> dict: # generate reply ####################################### debug_msg({'prompt': prompt, 'req_params': req_params}) stopping_strings = req_params.pop('stopping_strings', []) - logprob_proc = req_params.pop('logprob_proc', None) + logprob_proc = req_params.pop('logprob_proc', None) generator = generate_reply(prompt, req_params, stopping_strings=stopping_strings, is_chat=False) answer = '' @@ -286,9 +286,9 @@ def chat_completions(body: dict, is_legacy: bool=False) -> dict: "total_tokens": token_count + completion_token_count } } - if logprob_proc: # not official for chat yet + if logprob_proc: # not official for chat yet top_logprobs = convert_logprobs_to_tiktoken(model=requested_model, logprobs=logprob_proc.token_alternatives) - resp[resp_list][0]["logprobs"] = {'top_logprobs': [top_logprobs]} + resp[resp_list][0]["logprobs"] = {'top_logprobs': [top_logprobs]} # else: # resp[resp_list][0]["logprobs"] = None @@ -296,12 +296,12 @@ def chat_completions(body: dict, is_legacy: bool=False) -> dict: # generator -def stream_chat_completions(body: dict, is_legacy: bool=False): +def stream_chat_completions(body: dict, is_legacy: bool = False): # Chat Completions stream_object_type = 'chat.completions.chunk' created_time = int(time.time()) - cmpl_id = "chatcmpl-%d" % (int(time.time()*1000000000)) + cmpl_id = "chatcmpl-%d" % (int(time.time() * 1000000000)) resp_list = 'data' if is_legacy else 'choices' # common params @@ -309,7 +309,7 @@ def stream_chat_completions(body: dict, is_legacy: bool=False): req_params['stream'] = True requested_model = req_params.pop('requested_model') logprob_proc = req_params.pop('logprob_proc', None) - req_params['top_k'] = 20 # There is no best_of/top_k param for chat, but it is much improved with a higher top_k. + req_params['top_k'] = 20 # There is no best_of/top_k param for chat, but it is much improved with a higher top_k. # chat default max_tokens is 'inf', but also flexible max_tokens = 0 @@ -339,10 +339,10 @@ def stream_chat_completions(body: dict, is_legacy: bool=False): }], } - if logprob_proc: # not official for chat yet + if logprob_proc: # not official for chat yet top_logprobs = convert_logprobs_to_tiktoken(model=requested_model, logprobs=logprob_proc.token_alternatives) - chunk[resp_list][0]["logprobs"] = {'top_logprobs': [top_logprobs]} - #else: + chunk[resp_list][0]["logprobs"] = {'top_logprobs': [top_logprobs]} + # else: # chunk[resp_list][0]["logprobs"] = None return chunk @@ -350,7 +350,7 @@ def stream_chat_completions(body: dict, is_legacy: bool=False): # generate reply ####################################### debug_msg({'prompt': prompt, 'req_params': req_params}) - + stopping_strings = req_params.pop('stopping_strings', []) logprob_proc = req_params.pop('logprob_proc', None) @@ -377,9 +377,8 @@ def stream_chat_completions(body: dict, is_legacy: bool=False): completion_token_count += len(encode(new_content)[0]) chunk = chat_streaming_chunk(new_content) - - yield chunk + yield chunk stop_reason = "stop" if token_count + completion_token_count >= req_params['truncation_length'] or completion_token_count >= max_tokens: @@ -396,12 +395,12 @@ def stream_chat_completions(body: dict, is_legacy: bool=False): yield chunk -def completions(body: dict, is_legacy: bool=False): +def completions(body: dict, is_legacy: bool = False): # Legacy # Text Completions object_type = 'text_completion' created_time = int(time.time()) - cmpl_id = "conv-%d" % (int(time.time()*1000000000)) + cmpl_id = "conv-%d" % (int(time.time() * 1000000000)) resp_list = 'data' if is_legacy else 'choices' # ... encoded as a string, array of strings, array of tokens, or array of token arrays. @@ -433,7 +432,7 @@ def completions(body: dict, is_legacy: bool=False): if token_count + max_tokens > req_params['truncation_length']: err_msg = f"The token count of your prompt ({token_count}) plus max_tokens ({max_tokens}) cannot exceed the model's context length ({req_params['truncation_length']})." - #print(f"Warning: ${err_msg}") + # print(f"Warning: ${err_msg}") raise InvalidRequestError(message=err_msg, param=max_tokens_str) req_params['echo'] = default(body, 'echo', req_params['echo']) @@ -478,21 +477,21 @@ def completions(body: dict, is_legacy: bool=False): if logprob_proc: top_logprobs = convert_logprobs_to_tiktoken(model=requested_model, logprobs=logprob_proc.token_alternatives) - resp[resp_list][0]["logprobs"] = {'top_logprobs': [top_logprobs]} + resp[resp_list][0]["logprobs"] = {'top_logprobs': [top_logprobs]} else: - resp[resp_list][0]["logprobs"] = None + resp[resp_list][0]["logprobs"] = None return resp # generator -def stream_completions(body: dict, is_legacy: bool=False): +def stream_completions(body: dict, is_legacy: bool = False): # Legacy # Text Completions - #object_type = 'text_completion' + # object_type = 'text_completion' stream_object_type = 'text_completion.chunk' created_time = int(time.time()) - cmpl_id = "conv-%d" % (int(time.time()*1000000000)) + cmpl_id = "conv-%d" % (int(time.time() * 1000000000)) resp_list = 'data' if is_legacy else 'choices' # ... encoded as a string, array of strings, array of tokens, or array of token arrays. @@ -524,7 +523,7 @@ def stream_completions(body: dict, is_legacy: bool=False): if token_count + max_tokens > req_params['truncation_length']: err_msg = f"The token count of your prompt ({token_count}) plus max_tokens ({max_tokens}) cannot exceed the model's context length ({req_params['truncation_length']})." - #print(f"Warning: ${err_msg}") + # print(f"Warning: ${err_msg}") raise InvalidRequestError(message=err_msg, param=max_tokens_str) req_params['echo'] = default(body, 'echo', req_params['echo']) @@ -545,9 +544,9 @@ def stream_completions(body: dict, is_legacy: bool=False): } if logprob_proc: top_logprobs = convert_logprobs_to_tiktoken(model=requested_model, logprobs=logprob_proc.token_alternatives) - chunk[resp_list][0]["logprobs"] = {'top_logprobs': [top_logprobs]} + chunk[resp_list][0]["logprobs"] = {'top_logprobs': [top_logprobs]} else: - chunk[resp_list][0]["logprobs"] = None + chunk[resp_list][0]["logprobs"] = None return chunk @@ -583,7 +582,6 @@ def stream_completions(body: dict, is_legacy: bool=False): completion_token_count += len(encode(new_content)[0]) yield chunk - stop_reason = "stop" if token_count + completion_token_count >= req_params['truncation_length'] or completion_token_count >= max_tokens: stop_reason = "length" diff --git a/extensions/openai/defaults.py b/extensions/openai/defaults.py index 822a6e24..7c4f1c44 100644 --- a/extensions/openai/defaults.py +++ b/extensions/openai/defaults.py @@ -3,10 +3,10 @@ import copy # Slightly different defaults for OpenAI's API # Data type is important, Ex. use 0.0 for a float 0 default_req_params = { - 'max_new_tokens': 16, # 'Inf' for chat + 'max_new_tokens': 16, # 'Inf' for chat 'temperature': 1.0, 'top_p': 1.0, - 'top_k': 1, # choose 20 for chat in absence of another default + 'top_k': 1, # choose 20 for chat in absence of another default 'repetition_penalty': 1.18, 'repetition_penalty_range': 0, 'encoder_repetition_penalty': 1.0, @@ -15,7 +15,7 @@ default_req_params = { 'echo': False, 'seed': -1, # 'n' : default(body, 'n', 1), # 'n' doesn't have a direct map - 'truncation_length': 2048, # first use shared.settings value + 'truncation_length': 2048, # first use shared.settings value 'add_bos_token': True, 'do_sample': True, 'typical_p': 1.0, @@ -41,10 +41,13 @@ default_req_params = { # 'requested_model' - temporarily used } + def get_default_req_params(): return copy.deepcopy(default_req_params) # little helper to get defaults if arg is present but None and should be the same type as default. + + def default(dic, key, default): val = dic.get(key, default) if type(val) != type(default): @@ -59,6 +62,6 @@ def default(dic, key, default): val = default return val + def clamp(value, minvalue, maxvalue): return max(minvalue, min(value, maxvalue)) - diff --git a/extensions/openai/edits.py b/extensions/openai/edits.py index 3c51dc68..f10f5779 100644 --- a/extensions/openai/edits.py +++ b/extensions/openai/edits.py @@ -8,9 +8,9 @@ from extensions.openai.errors import * from modules.text_generation import encode, generate_reply -def edits(instruction: str, input: str, temperature = 1.0, top_p = 1.0) -> dict: +def edits(instruction: str, input: str, temperature=1.0, top_p=1.0) -> dict: - created_time = int(time.time()*1000) + created_time = int(time.time() * 1000) # Request parameters req_params = get_default_req_params() @@ -24,7 +24,7 @@ def edits(instruction: str, input: str, temperature = 1.0, top_p = 1.0) -> dict: ) instruction_template = default_template - + # Use the special instruction/input/response template for anything trained like Alpaca if shared.settings['instruction_template']: if 'Alpaca' in shared.settings['instruction_template']: @@ -41,7 +41,7 @@ def edits(instruction: str, input: str, temperature = 1.0, top_p = 1.0) -> dict: instruction_template = instruct.get('context', '') + template[:template.find('<|bot-message|>')].rstrip(' ') if instruct['user']: - stopping_strings.extend(['\n' + instruct['user'], instruct['user'] ]) + stopping_strings.extend(['\n' + instruct['user'], instruct['user']]) except Exception as e: instruction_template = default_template @@ -54,14 +54,14 @@ def edits(instruction: str, input: str, temperature = 1.0, top_p = 1.0) -> dict: edit_task = instruction_template.format(instruction=instruction, input=input) truncation_length = shared.settings['truncation_length'] - + token_count = len(encode(edit_task)[0]) max_tokens = truncation_length - token_count if max_tokens < 1: err_msg = f"This model maximum context length is {truncation_length} tokens. However, your messages resulted in over {truncation_length - max_tokens} tokens." raise InvalidRequestError(err_msg, param='input') - + req_params['max_new_tokens'] = max_tokens req_params['truncation_length'] = truncation_length req_params['temperature'] = temperature @@ -71,7 +71,7 @@ def edits(instruction: str, input: str, temperature = 1.0, top_p = 1.0) -> dict: req_params['custom_stopping_strings'] = shared.settings['custom_stopping_strings'] debug_msg({'edit_template': edit_task, 'req_params': req_params, 'token_count': token_count}) - + generator = generate_reply(edit_task, req_params, stopping_strings=stopping_strings, is_chat=False) longest_stop_len = max([len(x) for x in stopping_strings] + [0]) diff --git a/extensions/openai/embeddings.py b/extensions/openai/embeddings.py index 54e70ae7..c02bb933 100644 --- a/extensions/openai/embeddings.py +++ b/extensions/openai/embeddings.py @@ -6,26 +6,30 @@ from extensions.openai.errors import * st_model = os.environ["OPENEDAI_EMBEDDING_MODEL"] if "OPENEDAI_EMBEDDING_MODEL" in os.environ else "all-mpnet-base-v2" embeddings_model = None + def load_embedding_model(model): try: emb_model = SentenceTransformer(model) print(f"\nLoaded embedding model: {model}, max sequence length: {emb_model.max_seq_length}") except Exception as e: print(f"\nError: Failed to load embedding model: {model}") - raise ServiceUnavailableError(f"Error: Failed to load embedding model: {model}", internal_message = repr(e)) - + raise ServiceUnavailableError(f"Error: Failed to load embedding model: {model}", internal_message=repr(e)) + return emb_model + def get_embeddings_model(): global embeddings_model, st_model if st_model and not embeddings_model: - embeddings_model = load_embedding_model(st_model) # lazy load the model + embeddings_model = load_embedding_model(st_model) # lazy load the model return embeddings_model + def get_embeddings_model_name(): global st_model return st_model + def embeddings(input: list, encoding_format: str): embeddings = get_embeddings_model().encode(input).tolist() @@ -47,4 +51,4 @@ def embeddings(input: list, encoding_format: str): debug_msg(f"Embeddings return size: {len(embeddings[0])}, number: {len(embeddings)}") - return response \ No newline at end of file + return response diff --git a/extensions/openai/errors.py b/extensions/openai/errors.py index df66a300..ff519c4f 100644 --- a/extensions/openai/errors.py +++ b/extensions/openai/errors.py @@ -1,8 +1,9 @@ class OpenAIError(Exception): - def __init__(self, message = None, code = 500, internal_message = ''): + def __init__(self, message=None, code=500, internal_message=''): self.message = message self.code = code self.internal_message = internal_message + def __repr__(self): return "%s(message=%r, code=%d)" % ( self.__class__.__name__, @@ -10,10 +11,12 @@ class OpenAIError(Exception): self.code, ) + class InvalidRequestError(OpenAIError): - def __init__(self, message, param, code = 400, error_type ='InvalidRequestError', internal_message = ''): + def __init__(self, message, param, code=400, error_type='InvalidRequestError', internal_message=''): super(OpenAIError, self).__init__(message, code, error_type, internal_message) self.param = param + def __repr__(self): return "%s(message=%r, code=%d, param=%s)" % ( self.__class__.__name__, @@ -22,6 +25,7 @@ class InvalidRequestError(OpenAIError): self.param, ) + class ServiceUnavailableError(OpenAIError): - def __init__(self, message = None, code = 500, error_type ='ServiceUnavailableError', internal_message = ''): + def __init__(self, message=None, code=500, error_type='ServiceUnavailableError', internal_message=''): super(OpenAIError, self).__init__(message, code, error_type, internal_message) diff --git a/extensions/openai/images.py b/extensions/openai/images.py index d911ed63..d2be3192 100644 --- a/extensions/openai/images.py +++ b/extensions/openai/images.py @@ -3,6 +3,7 @@ import time import requests from extensions.openai.errors import * + def generations(prompt: str, size: str, response_format: str, n: int): # Stable Diffusion callout wrapper for txt2img # Low effort implementation for compatibility. With only "prompt" being passed and assuming DALL-E @@ -15,7 +16,7 @@ def generations(prompt: str, size: str, response_format: str, n: int): # require changing the form data handling to accept multipart form data, also to properly support # url return types will require file management and a web serving files... Perhaps later! - width, height = [ int(x) for x in size.split('x') ] # ignore the restrictions on size + width, height = [int(x) for x in size.split('x')] # ignore the restrictions on size # to hack on better generation, edit default payload. payload = { @@ -23,7 +24,7 @@ def generations(prompt: str, size: str, response_format: str, n: int): 'width': width, 'height': height, 'batch_size': n, - 'restore_faces': True, # slightly less horrible + 'restore_faces': True, # slightly less horrible } resp = { @@ -37,7 +38,7 @@ def generations(prompt: str, size: str, response_format: str, n: int): response = requests.post(url=sd_url, json=payload) r = response.json() if response.status_code != 200 or 'images' not in r: - raise ServiceUnavailableError(r.get('detail', [{'msg': 'Unknown error calling Stable Diffusion'}])[0]['msg'], code = response.status_code) + raise ServiceUnavailableError(r.get('detail', [{'msg': 'Unknown error calling Stable Diffusion'}])[0]['msg'], code=response.status_code) # r['parameters']... for b64_json in r['images']: if response_format == 'b64_json': @@ -45,4 +46,4 @@ def generations(prompt: str, size: str, response_format: str, n: int): else: resp['data'].extend([{'url': f'data:image/png;base64,{b64_json}'}]) # yeah it's lazy. requests.get() will not work with this - return resp \ No newline at end of file + return resp diff --git a/extensions/openai/models.py b/extensions/openai/models.py index bed5ed49..035996f6 100644 --- a/extensions/openai/models.py +++ b/extensions/openai/models.py @@ -7,15 +7,18 @@ from modules.models_settings import (get_model_settings_from_yamls, from extensions.openai.embeddings import get_embeddings_model_name from extensions.openai.errors import * + def get_current_model_list() -> list: - return [ shared.model_name ] # The real chat/completions model, maybe "None" + return [shared.model_name] # The real chat/completions model, maybe "None" + def get_pseudo_model_list() -> list: - return [ # these are expected by so much, so include some here as a dummy + return [ # these are expected by so much, so include some here as a dummy 'gpt-3.5-turbo', 'text-embedding-ada-002', ] + def load_model(model_name: str) -> dict: resp = { "id": model_name, @@ -23,7 +26,7 @@ def load_model(model_name: str) -> dict: "owner": "self", "ready": True, } - if model_name not in get_pseudo_model_list() + [ get_embeddings_model_name() ] + get_current_model_list(): # Real model only + if model_name not in get_pseudo_model_list() + [get_embeddings_model_name()] + get_current_model_list(): # Real model only # No args. Maybe it works anyways! # TODO: hack some heuristics into args for better results @@ -39,7 +42,7 @@ def load_model(model_name: str) -> dict: shared.model, shared.tokenizer = load_model(shared.model_name) - if not shared.model: # load failed. + if not shared.model: # load failed. shared.model_name = "None" raise OpenAIError(f"Model load failed for: {shared.model_name}") @@ -48,16 +51,16 @@ def load_model(model_name: str) -> dict: def list_models(is_legacy: bool = False) -> dict: # TODO: Lora's? - all_model_list = get_current_model_list() + [ get_embeddings_model_name() ] + get_pseudo_model_list() + get_available_models() + all_model_list = get_current_model_list() + [get_embeddings_model_name()] + get_pseudo_model_list() + get_available_models() models = {} if is_legacy: - models = [{ "id": id, "object": "engine", "owner": "user", "ready": True } for id in all_model_list ] + models = [{"id": id, "object": "engine", "owner": "user", "ready": True} for id in all_model_list] if not shared.model: models[0]['ready'] = False else: - models = [{ "id": id, "object": "model", "owned_by": "user", "permission": [] } for id in all_model_list ] + models = [{"id": id, "object": "model", "owned_by": "user", "permission": []} for id in all_model_list] resp = { "object": "list", @@ -74,4 +77,3 @@ def model_info(model_name: str) -> dict: "owned_by": "user", "permission": [] } - diff --git a/extensions/openai/moderations.py b/extensions/openai/moderations.py index 7daf0176..66dfec9f 100644 --- a/extensions/openai/moderations.py +++ b/extensions/openai/moderations.py @@ -4,10 +4,10 @@ from numpy.linalg import norm from extensions.openai.embeddings import get_embeddings_model -moderations_disabled = False # return 0/false +moderations_disabled = False # return 0/false category_embeddings = None antonym_embeddings = None -categories = [ "sexual", "hate", "harassment", "self-harm", "sexual/minors", "hate/threatening", "violence/graphic", "self-harm/intent", "self-harm/instructions", "harassment/threatening", "violence" ] +categories = ["sexual", "hate", "harassment", "self-harm", "sexual/minors", "hate/threatening", "violence/graphic", "self-harm/intent", "self-harm/instructions", "harassment/threatening", "violence"] flag_threshold = 0.5 @@ -40,23 +40,22 @@ def moderations(input): embeddings_model = get_embeddings_model() if not embeddings_model or moderations_disabled: results['results'] = [{ - 'categories': dict([ (C, False) for C in categories]), - 'category_scores': dict([ (C, 0.0) for C in categories]), + 'categories': dict([(C, False) for C in categories]), + 'category_scores': dict([(C, 0.0) for C in categories]), 'flagged': False, }] return results category_embeddings = get_category_embeddings() - # input, string or array if isinstance(input, str): input = [input] for in_str in input: for ine in embeddings_model.encode([in_str]).tolist(): - category_scores = dict([ (C, mod_score(category_embeddings[C], ine)) for C in categories ]) - category_flags = dict([ (C, bool(category_scores[C] > flag_threshold)) for C in categories ]) + category_scores = dict([(C, mod_score(category_embeddings[C], ine)) for C in categories]) + category_flags = dict([(C, bool(category_scores[C] > flag_threshold)) for C in categories]) flagged = any(category_flags.values()) results['results'].extend([{ @@ -67,4 +66,4 @@ def moderations(input): print(results) - return results \ No newline at end of file + return results diff --git a/extensions/openai/script.py b/extensions/openai/script.py index 1ff1eb6b..a0a5bcf6 100644 --- a/extensions/openai/script.py +++ b/extensions/openai/script.py @@ -22,6 +22,7 @@ params = { 'port': int(os.environ.get('OPENEDAI_PORT')) if 'OPENEDAI_PORT' in os.environ else 5001, } + class Handler(BaseHTTPRequestHandler): def send_access_control_headers(self): self.send_header("Access-Control-Allow-Origin", "*") @@ -72,8 +73,8 @@ class Handler(BaseHTTPRequestHandler): if not no_debug: debug_msg(r_utf8) - def openai_error(self, message, code = 500, error_type = 'APIError', param = '', internal_message = ''): - + def openai_error(self, message, code=500, error_type='APIError', param='', internal_message=''): + error_resp = { 'error': { 'message': message, @@ -84,10 +85,10 @@ class Handler(BaseHTTPRequestHandler): } if internal_message: print(internal_message) - #error_resp['internal_message'] = internal_message + # error_resp['internal_message'] = internal_message self.return_json(error_resp, code) - + def openai_error_handler(func): def wrapper(self): try: @@ -156,7 +157,7 @@ class Handler(BaseHTTPRequestHandler): response = OAIcompletions.stream_chat_completions(body, is_legacy=is_legacy) else: response = OAIcompletions.stream_completions(body, is_legacy=is_legacy) - + for resp in response: self.send_sse(resp) @@ -182,7 +183,7 @@ class Handler(BaseHTTPRequestHandler): instruction = body['instruction'] input = body.get('input', '') - temperature = clamp(default(body, 'temperature', req_params['temperature']), 0.001, 1.999) # fixup absolute 0.0 + temperature = clamp(default(body, 'temperature', req_params['temperature']), 0.001, 1.999) # fixup absolute 0.0 top_p = clamp(default(body, 'top_p', req_params['top_p']), 0.001, 1.0) response = OAIedits.edits(instruction, input, temperature, top_p) @@ -205,7 +206,7 @@ class Handler(BaseHTTPRequestHandler): input = body.get('input', body.get('text', '')) if not input: raise InvalidRequestError("Missing required argument input", params='input') - + if type(input) is str: input = [input] @@ -225,15 +226,15 @@ class Handler(BaseHTTPRequestHandler): elif self.path == '/api/v1/token-count': # NOT STANDARD. lifted from the api extension, but it's still very useful to calculate tokenized length client side. response = token_count(body['prompt']) - + self.return_json(response, no_debug=True) elif self.path == '/api/v1/token/encode': # NOT STANDARD. needed to support logit_bias, logprobs and token arrays for native models encoding_format = body.get('encoding_format', '') - + response = token_encode(body['input'], encoding_format) - + self.return_json(response, no_debug=True) elif self.path == '/api/v1/token/decode': @@ -241,7 +242,7 @@ class Handler(BaseHTTPRequestHandler): encoding_format = body.get('encoding_format', '') response = token_decode(body['input'], encoding_format) - + self.return_json(response, no_debug=True) else: diff --git a/extensions/openai/tokens.py b/extensions/openai/tokens.py index 9f0208d3..f243c3c9 100644 --- a/extensions/openai/tokens.py +++ b/extensions/openai/tokens.py @@ -1,6 +1,7 @@ from extensions.openai.utils import float_list_to_base64 from modules.text_generation import encode, decode + def token_count(prompt): tokens = encode(prompt)[0] @@ -11,8 +12,8 @@ def token_count(prompt): } -def token_encode(input, encoding_format = ''): - #if isinstance(input, list): +def token_encode(input, encoding_format=''): + # if isinstance(input, list): tokens = encode(input)[0] return { @@ -25,9 +26,9 @@ def token_encode(input, encoding_format = ''): def token_decode(tokens, encoding_format): - #if isinstance(input, list): -# if encoding_format == "base64": -# tokens = base64_to_float_list(tokens) + # if isinstance(input, list): + # if encoding_format == "base64": + # tokens = base64_to_float_list(tokens) output = decode(tokens)[0] return { diff --git a/extensions/openai/utils.py b/extensions/openai/utils.py index d01ec14d..0c9441a3 100644 --- a/extensions/openai/utils.py +++ b/extensions/openai/utils.py @@ -2,6 +2,7 @@ import os import base64 import numpy as np + def float_list_to_base64(float_list): # Convert the list to a float32 array that the OpenAPI client expects float_array = np.array(float_list, dtype="float32") @@ -16,11 +17,13 @@ def float_list_to_base64(float_list): ascii_string = encoded_bytes.decode('ascii') return ascii_string + def end_line(s): if s and s[-1] != '\n': s = s + '\n' return s + def debug_msg(*args, **kwargs): if 'OPENEDAI_DEBUG' in os.environ: - print(*args, **kwargs) \ No newline at end of file + print(*args, **kwargs) diff --git a/extensions/sd_api_pictures/script.py b/extensions/sd_api_pictures/script.py index 78488cd0..d7f0fa69 100644 --- a/extensions/sd_api_pictures/script.py +++ b/extensions/sd_api_pictures/script.py @@ -126,6 +126,8 @@ def input_modifier(string): return string # Get and save the Stable Diffusion-generated picture + + def get_SD_pictures(description, character): global params @@ -186,6 +188,8 @@ def get_SD_pictures(description, character): # TODO: how do I make the UI history ignore the resulting pictures (I don't want HTML to appear in history) # and replace it with 'text' for the purposes of logging? + + def output_modifier(string, state): """ This function is applied to the model outputs. diff --git a/extensions/superbooga/script.py b/extensions/superbooga/script.py index f67a956e..5ef14d9d 100644 --- a/extensions/superbooga/script.py +++ b/extensions/superbooga/script.py @@ -113,7 +113,7 @@ def custom_generate_chat_prompt(user_input, state, **kwargs): if len(history['internal']) > params['chunk_count'] and user_input != '': chunks = [] hist_size = len(history['internal']) - for i in range(hist_size-1): + for i in range(hist_size - 1): chunks.append(make_single_exchange(i)) add_chunks_to_collector(chunks, chat_collector) diff --git a/extensions/whisper_stt/script.py b/extensions/whisper_stt/script.py index 44a9ac81..1e07ad2c 100644 --- a/extensions/whisper_stt/script.py +++ b/extensions/whisper_stt/script.py @@ -16,7 +16,7 @@ params = { } -def do_stt(audio,whipser_model,whipser_language): +def do_stt(audio, whipser_model, whipser_language): transcription = "" r = sr.Recognizer() @@ -33,10 +33,10 @@ def do_stt(audio,whipser_model,whipser_language): return transcription -def auto_transcribe(audio, auto_submit,whipser_model,whipser_language): +def auto_transcribe(audio, auto_submit, whipser_model, whipser_language): if audio is None: return "", "" - transcription = do_stt(audio,whipser_model,whipser_language) + transcription = do_stt(audio, whipser_model, whipser_language) if auto_submit: input_hijack.update({"state": True, "value": [transcription, transcription]}) @@ -50,11 +50,11 @@ def ui(): with gr.Row(): with gr.Accordion("Settings", open=False): auto_submit = gr.Checkbox(label='Submit the transcribed audio automatically', value=params['auto_submit']) - whipser_model = gr.Dropdown(label='Whisper Model', value=params['whipser_model'],choices=["tiny.en","base.en", "small.en","medium.en","tiny","base","small","medium","large"]) - whipser_language = gr.Dropdown(label='Whisper Language', value=params['whipser_language'],choices=["chinese","german","spanish","russian","korean","french","japanese","portuguese","turkish","polish","catalan","dutch","arabic","swedish","italian","indonesian","hindi","finnish","vietnamese","hebrew","ukrainian","greek","malay","czech","romanian","danish","hungarian","tamil","norwegian","thai","urdu","croatian","bulgarian","lithuanian","latin","maori","malayalam","welsh","slovak","telugu","persian","latvian","bengali","serbian","azerbaijani","slovenian","kannada","estonian","macedonian","breton","basque","icelandic","armenian","nepali","mongolian","bosnian","kazakh","albanian","swahili","galician","marathi","punjabi","sinhala","khmer","shona","yoruba","somali","afrikaans","occitan","georgian","belarusian","tajik","sindhi","gujarati","amharic","yiddish","lao","uzbek","faroese","haitian creole","pashto","turkmen","nynorsk","maltese","sanskrit","luxembourgish","myanmar","tibetan","tagalog","malagasy","assamese","tatar","hawaiian","lingala","hausa","bashkir","javanese","sundanese"]) + whipser_model = gr.Dropdown(label='Whisper Model', value=params['whipser_model'], choices=["tiny.en", "base.en", "small.en", "medium.en", "tiny", "base", "small", "medium", "large"]) + whipser_language = gr.Dropdown(label='Whisper Language', value=params['whipser_language'], choices=["chinese", "german", "spanish", "russian", "korean", "french", "japanese", "portuguese", "turkish", "polish", "catalan", "dutch", "arabic", "swedish", "italian", "indonesian", "hindi", "finnish", "vietnamese", "hebrew", "ukrainian", "greek", "malay", "czech", "romanian", "danish", "hungarian", "tamil", "norwegian", "thai", "urdu", "croatian", "bulgarian", "lithuanian", "latin", "maori", "malayalam", "welsh", "slovak", "telugu", "persian", "latvian", "bengali", "serbian", "azerbaijani", "slovenian", "kannada", "estonian", "macedonian", "breton", "basque", "icelandic", "armenian", "nepali", "mongolian", "bosnian", "kazakh", "albanian", "swahili", "galician", "marathi", "punjabi", "sinhala", "khmer", "shona", "yoruba", "somali", "afrikaans", "occitan", "georgian", "belarusian", "tajik", "sindhi", "gujarati", "amharic", "yiddish", "lao", "uzbek", "faroese", "haitian creole", "pashto", "turkmen", "nynorsk", "maltese", "sanskrit", "luxembourgish", "myanmar", "tibetan", "tagalog", "malagasy", "assamese", "tatar", "hawaiian", "lingala", "hausa", "bashkir", "javanese", "sundanese"]) audio.change( - auto_transcribe, [audio, auto_submit,whipser_model,whipser_language], [shared.gradio['textbox'], audio]).then( + auto_transcribe, [audio, auto_submit, whipser_model, whipser_language], [shared.gradio['textbox'], audio]).then( None, auto_submit, None, _js="(check) => {if (check) { document.getElementById('Generate').click() }}") whipser_model.change(lambda x: params.update({"whipser_model": x}), whipser_model, None) whipser_language.change(lambda x: params.update({"whipser_language": x}), whipser_language, None) diff --git a/modules/LoRA.py b/modules/LoRA.py index 2eade078..0626c969 100644 --- a/modules/LoRA.py +++ b/modules/LoRA.py @@ -66,7 +66,7 @@ def add_lora_autogptq(lora_names): logger.error("This version of AutoGPTQ does not support LoRA. You need to install from source or wait for a new release.") return - if len(lora_names) == 0: + if len(lora_names) == 0: reload_model() shared.lora_names = [] @@ -108,14 +108,14 @@ def add_lora_transformers(lora_names): # If any LoRA needs to be removed, start over if len(removed_set) > 0: # shared.model may no longer be PeftModel - if hasattr(shared.model, 'disable_adapter'): - shared.model.disable_adapter() + if hasattr(shared.model, 'disable_adapter'): + shared.model.disable_adapter() shared.model = shared.model.base_model.model if len(lora_names) > 0: params = {} if not shared.args.cpu: - if shared.args.load_in_4bit or shared.args.load_in_8bit: + if shared.args.load_in_4bit or shared.args.load_in_8bit: params['peft_type'] = shared.model.dtype else: params['dtype'] = shared.model.dtype diff --git a/modules/loaders.py b/modules/loaders.py index e0db482c..bb539564 100644 --- a/modules/loaders.py +++ b/modules/loaders.py @@ -54,14 +54,14 @@ loaders_and_params = { 'trust_remote_code', 'transformers_info' ], - 'ExLlama' : [ + 'ExLlama': [ 'gpu_split', 'max_seq_len', 'compress_pos_emb', 'alpha_value', 'exllama_info', ], - 'ExLlama_HF' : [ + 'ExLlama_HF': [ 'gpu_split', 'max_seq_len', 'compress_pos_emb', diff --git a/modules/models.py b/modules/models.py index 497d6b78..2029e3a0 100644 --- a/modules/models.py +++ b/modules/models.py @@ -106,11 +106,11 @@ def load_tokenizer(model_name, model): use_fast=False ) except ValueError: - tokenizer = AutoTokenizer.from_pretrained( + tokenizer = AutoTokenizer.from_pretrained( path_to_model, trust_remote_code=shared.args.trust_remote_code, use_fast=True - ) + ) if tokenizer.__class__.__name__ == 'LlamaTokenizer': pairs = [ diff --git a/modules/sampler_hijack.py b/modules/sampler_hijack.py index 391ece92..0a86b4fd 100644 --- a/modules/sampler_hijack.py +++ b/modules/sampler_hijack.py @@ -126,6 +126,7 @@ class RepetitionPenaltyLogitsProcessorWithRange(LogitsProcessor): ''' Copied from the transformers library ''' + def __init__(self, penalty: float, _range: int): if not isinstance(penalty, float) or not (penalty > 0): raise ValueError(f"`penalty` has to be a strictly positive float, but is {penalty}") diff --git a/modules/shared.py b/modules/shared.py index 14a10e57..b54f9aac 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -181,7 +181,7 @@ parser.add_argument("--gradio-auth-path", type=str, help='Set the gradio authent # API parser.add_argument('--api', action='store_true', help='Enable the API extension.') parser.add_argument('--api-blocking-port', type=int, default=5000, help='The listening port for the blocking API.') -parser.add_argument('--api-streaming-port', type=int, default=5005, help='The listening port for the streaming API.') +parser.add_argument('--api-streaming-port', type=int, default=5005, help='The listening port for the streaming API.') parser.add_argument('--public-api', action='store_true', help='Create a public URL for the API using Cloudfare.') # Multimodal diff --git a/modules/utils.py b/modules/utils.py index 8b662be1..e257de2d 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -116,7 +116,7 @@ def get_available_loras(): def get_datasets(path: str, ext: str): # include subdirectories for raw txt files to allow training from a subdirectory of txt files if ext == "txt": - return ['None'] + sorted(set([k.stem for k in list(Path(path).glob('txt'))+list(Path(path).glob('*/')) if k.stem != 'put-trainer-datasets-here']), key=natural_keys) + return ['None'] + sorted(set([k.stem for k in list(Path(path).glob('txt')) + list(Path(path).glob('*/')) if k.stem != 'put-trainer-datasets-here']), key=natural_keys) return ['None'] + sorted(set([k.stem for k in Path(path).glob(f'*.{ext}') if k.stem != 'put-trainer-datasets-here']), key=natural_keys) From 2463d7c098da47f2a0f6f6d6ec5268cd905628d4 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Wed, 12 Jul 2023 11:35:43 -0700 Subject: [PATCH 47/54] Spaces --- extensions/sd_api_pictures/script.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/extensions/sd_api_pictures/script.py b/extensions/sd_api_pictures/script.py index d7f0fa69..78488cd0 100644 --- a/extensions/sd_api_pictures/script.py +++ b/extensions/sd_api_pictures/script.py @@ -126,8 +126,6 @@ def input_modifier(string): return string # Get and save the Stable Diffusion-generated picture - - def get_SD_pictures(description, character): global params @@ -188,8 +186,6 @@ def get_SD_pictures(description, character): # TODO: how do I make the UI history ignore the resulting pictures (I don't want HTML to appear in history) # and replace it with 'text' for the purposes of logging? - - def output_modifier(string, state): """ This function is applied to the model outputs. From c592a9b740552442d33dfa0a74817c50b2f35c91 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Wed, 12 Jul 2023 13:33:44 -0700 Subject: [PATCH 48/54] Fix #3117 --- server.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/server.py b/server.py index 2723b284..05455775 100644 --- a/server.py +++ b/server.py @@ -146,11 +146,6 @@ def download_model_wrapper(repo_id, progress=gr.Progress()): yield ("Getting the output folder") models_dir = Path(shared.args.model_dir) - - # If the last part of the path is "models", remove it - if models_dir.name.lower() == 'models': - models_dir = models_dir.parent - output_folder = downloader.get_output_folder(model, branch, is_lora, base_folder=models_dir) if check: From d0a626f32f4121e66c68998140cf97b3cd7081df Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Wed, 12 Jul 2023 13:54:43 -0700 Subject: [PATCH 49/54] Change reload screen color --- server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server.py b/server.py index 05455775..6f1834d6 100644 --- a/server.py +++ b/server.py @@ -848,7 +848,7 @@ def create_interface(): # Reset interface event shared.gradio['reset_interface'].click( set_interface_arguments, gradio('interface_modes_menu', 'extensions_menu', 'bool_menu'), None).then( - lambda: None, None, None, _js='() => {document.body.innerHTML=\'

Reloading...

\'; setTimeout(function(){location.reload()},2500); return []}') + lambda: None, None, None, _js='() => {document.body.innerHTML=\'

Reloading...

\'; setTimeout(function(){location.reload()},250000); return []}') shared.gradio['toggle_dark_mode'].click(lambda: None, None, None, _js='() => {document.getElementsByTagName("body")[0].classList.toggle("dark")}') From eb823fce96000df98ed8df8634848125fe5ecf1c Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Wed, 12 Jul 2023 13:55:19 -0700 Subject: [PATCH 50/54] Fix typo --- server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server.py b/server.py index 6f1834d6..5756f0ff 100644 --- a/server.py +++ b/server.py @@ -848,7 +848,7 @@ def create_interface(): # Reset interface event shared.gradio['reset_interface'].click( set_interface_arguments, gradio('interface_modes_menu', 'extensions_menu', 'bool_menu'), None).then( - lambda: None, None, None, _js='() => {document.body.innerHTML=\'

Reloading...

\'; setTimeout(function(){location.reload()},250000); return []}') + lambda: None, None, None, _js='() => {document.body.innerHTML=\'

Reloading...

\'; setTimeout(function(){location.reload()},2500); return []}') shared.gradio['toggle_dark_mode'].click(lambda: None, None, None, _js='() => {document.getElementsByTagName("body")[0].classList.toggle("dark")}') From 0e6295886d2d0e9309c8223201e8a4bd55e4735c Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Wed, 12 Jul 2023 14:19:33 -0700 Subject: [PATCH 51/54] Fix lora download folder --- server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server.py b/server.py index 05455775..56f94963 100644 --- a/server.py +++ b/server.py @@ -145,8 +145,8 @@ def download_model_wrapper(repo_id, progress=gr.Progress()): links, sha256, is_lora = downloader.get_download_links_from_huggingface(model, branch, text_only=False) yield ("Getting the output folder") - models_dir = Path(shared.args.model_dir) - output_folder = downloader.get_output_folder(model, branch, is_lora, base_folder=models_dir) + base_folder = shared.args.lora_dir if is_lora else shared.args.model_dir + output_folder = downloader.get_output_folder(model, branch, is_lora, base_folder=base_folder) if check: progress(0.5) From 6d1e9115774a8b0adcbda38786af3856fe57e452 Mon Sep 17 00:00:00 2001 From: Morgan Schweers Date: Thu, 13 Jul 2023 13:22:41 -0700 Subject: [PATCH 52/54] Add support for logits processors in extensions (#3029) --- modules/extensions.py | 13 +++++++++++-- modules/text_generation.py | 8 ++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/modules/extensions.py b/modules/extensions.py index 8705101a..faf6cf6d 100644 --- a/modules/extensions.py +++ b/modules/extensions.py @@ -106,15 +106,23 @@ def _apply_history_modifier_extensions(history): return history -# Extension functions that override the default tokenizer output - currently only the first one will work +# Extension functions that override the default tokenizer output - The order of execution is not defined def _apply_tokenizer_extensions(function_name, state, prompt, input_ids, input_embeds): for extension, _ in iterator(): if hasattr(extension, function_name): - return getattr(extension, function_name)(state, prompt, input_ids, input_embeds) + prompt, input_ids, input_embeds = getattr(extension, function_name)(state, prompt, input_ids, input_embeds) return prompt, input_ids, input_embeds +# Allow extensions to add their own logits processors to the stack being run. +# Each extension would call `processor_list.append({their LogitsProcessor}())`. +def _apply_logits_processor_extensions(function_name, processor_list, input_ids): + for extension, _ in iterator(): + if hasattr(extension, function_name): + getattr(extension, function_name)(processor_list, input_ids) + + # Get prompt length in tokens after applying extension functions which override the default tokenizer output # currently only the first one will work def _apply_custom_tokenized_length(prompt): @@ -183,6 +191,7 @@ EXTENSION_MAP = { "history": _apply_history_modifier_extensions, "bot_prefix": partial(_apply_string_extensions, "bot_prefix_modifier"), "tokenizer": partial(_apply_tokenizer_extensions, "tokenizer_modifier"), + 'logits_processor': partial(_apply_logits_processor_extensions, 'logits_processor_modifier'), "input_hijack": _apply_input_hijack, "custom_generate_chat_prompt": _apply_custom_generate_chat_prompt, "custom_generate_reply": _apply_custom_generate_reply, diff --git a/modules/text_generation.py b/modules/text_generation.py index b7f6edf3..566c2f55 100644 --- a/modules/text_generation.py +++ b/modules/text_generation.py @@ -8,6 +8,7 @@ import traceback import numpy as np import torch import transformers +from transformers import LogitsProcessorList import modules.shared as shared from modules.callbacks import ( @@ -264,6 +265,13 @@ def generate_reply_HF(question, original_question, seed, state, stopping_strings generate_params['stopping_criteria'] = transformers.StoppingCriteriaList() generate_params['stopping_criteria'].append(_StopEverythingStoppingCriteria()) + processor = state.get('logits_processor', LogitsProcessorList([])) + # In case folks just pass in a processor by itself. + if type(processor) != LogitsProcessorList: + processor = LogitsProcessorList([processor]) + apply_extensions('logits_processor', processor, input_ids) + generate_params['logits_processor'] = processor + t0 = time.time() try: if not is_chat and not shared.is_seq2seq: From 9800745db992e772da51fdb97d404464f7bac064 Mon Sep 17 00:00:00 2001 From: SeanScripts <64337075+SeanScripts@users.noreply.github.com> Date: Thu, 13 Jul 2023 15:30:22 -0500 Subject: [PATCH 53/54] Color tokens by probability and/or perplexity (#3078) --- extensions/perplexity_colors/script.py | 215 +++++++++++++++++++++++++ 1 file changed, 215 insertions(+) create mode 100644 extensions/perplexity_colors/script.py diff --git a/extensions/perplexity_colors/script.py b/extensions/perplexity_colors/script.py new file mode 100644 index 00000000..84b62a30 --- /dev/null +++ b/extensions/perplexity_colors/script.py @@ -0,0 +1,215 @@ +import gradio +import torch +from transformers import LogitsProcessor +import numpy as np + +from modules import shared + +params = { + 'color_by_perplexity': False, + 'color_by_probability': False, + 'ppl_scale': 15.0, # No slider for this right now, because I don't think it really needs to be changed. Very large perplexity scores don't show up often. + #'probability_dropdown': False +} + +class PerplexityLogits(LogitsProcessor): + def __init__(self, verbose=False): + self.generated_token_ids = [] + self.selected_probs = [] + self.top_token_ids_list = [] + self.top_probs_list = [] + self.perplexities_list = [] + self.last_probs = None + self.verbose = verbose + + def __call__(self, input_ids, scores): + probs = torch.softmax(scores, dim=-1, dtype=torch.float) + log_probs = torch.nan_to_num(torch.log(probs)) + entropy = -torch.sum(probs*log_probs) + entropy = entropy.cpu().numpy() + perplexity = round(float(np.exp(entropy)), 4) + self.perplexities_list.append(perplexity) + last_token_id = int(input_ids[0][-1].cpu().numpy().item()) + # Store the generated tokens (not sure why this isn't accessible in the output endpoint!) + self.generated_token_ids.append(last_token_id) + # Get last probability, and add to the list if it wasn't there + if len(self.selected_probs) > 0: + # Is the selected token in the top tokens? + if self.verbose: + print(shared.tokenizer.decode(last_token_id)) + print([shared.tokenizer.decode(token_id) for token_id in self.top_token_ids_list[-1]]) + print(self.top_probs_list[-1]) + if last_token_id in self.top_token_ids_list[-1]: + idx = self.top_token_ids_list[-1].index(last_token_id) + self.selected_probs.append(self.top_probs_list[-1][idx]) + else: + self.top_token_ids_list[-1].append(last_token_id) + last_prob = round(float(self.last_probs[last_token_id]), 4) + self.top_probs_list[-1].append(last_prob) + self.selected_probs.append(last_prob) + else: + self.selected_probs.append(1.0) # Placeholder for the last token of the prompt + + if self.verbose: + pplbar = "-" + if not np.isnan(perplexity): + pplbar = "*"*round(perplexity) + print(f"{last_token}\t{perplexity:.2f}\t{pplbar}") + + # Get top 5 probabilities + top_tokens_and_probs = torch.topk(probs, 5) + top_probs = top_tokens_and_probs.values.cpu().numpy().astype(float).tolist() + top_token_ids = top_tokens_and_probs.indices.cpu().numpy().astype(int).tolist() + + self.top_token_ids_list.append(top_token_ids) + self.top_probs_list.append(top_probs) + + probs = probs.cpu().numpy().flatten() + self.last_probs = probs # Need to keep this as a reference for top probs + + # Doesn't actually modify the logits! + return scores + +# Stores the perplexity and top probabilities +ppl_logits_processor = None + +def logits_processor_modifier(logits_processor_list, input_ids): + global ppl_logits_processor + ppl_logits_processor = PerplexityLogits() + logits_processor_list.append(ppl_logits_processor) + +def output_modifier(text): + global ppl_logits_processor + + # TODO: It's probably more efficient to do this above rather than modifying all these lists + # Remove last element of perplexities_list, top_token_ids_list, top_tokens_list, top_probs_list since everything is off by one because this extension runs before generation + perplexities = ppl_logits_processor.perplexities_list[:-1] + top_token_ids_list = ppl_logits_processor.top_token_ids_list[:-1] + top_tokens_list = [[shared.tokenizer.decode(token_id) for token_id in top_token_ids] for top_token_ids in top_token_ids_list] + top_probs_list = ppl_logits_processor.top_probs_list[:-1] + # Remove first element of generated_token_ids, generated_tokens, selected_probs because they are for the last token of the prompt + gen_token_ids = ppl_logits_processor.generated_token_ids[1:] + gen_tokens = [shared.tokenizer.decode(token_id) for token_id in gen_token_ids] + sel_probs = ppl_logits_processor.selected_probs[1:] + + end_part = '' # Helps with finding the index after replacing part of the text. + in_code = False # Since the tags mess up code blocks, avoid coloring while inside a code block, based on finding tokens with '`' in them + + if params['color_by_probability'] and params['color_by_perplexity']: + i = 0 + for token, prob, ppl, top_tokens, top_probs in zip(gen_tokens, sel_probs, perplexities, top_tokens_list, top_probs_list): + if '`' in token: + in_code = not in_code + continue + if in_code: + continue + color = probability_perplexity_color_scale(prob, ppl) + if token in text[i:]: + text = text[:i] + text[i:].replace(token, add_color_html(token, color), 1) + i += text[i:].find(end_part) + len(end_part) + elif params['color_by_perplexity']: + i = 0 + for token, ppl, top_tokens, top_probs in zip(gen_tokens, perplexities, top_tokens_list, top_probs_list): + if '`' in token: + in_code = not in_code + continue + if in_code: + continue + color = perplexity_color_scale(ppl) + if token in text[i:]: + text = text[:i] + text[i:].replace(token, add_color_html(token, color), 1) + i += text[i:].find(end_part) + len(end_part) + elif params['color_by_probability']: + i = 0 + for token, prob, top_tokens, top_probs in zip(gen_tokens, sel_probs, top_tokens_list, top_probs_list): + if '`' in token: + in_code = not in_code + continue + if in_code: + continue + color = probability_color_scale(prob) + if token in text[i:]: + text = text[:i] + text[i:].replace(token, add_color_html(token, color), 1) + i += text[i:].find(end_part) + len(end_part) + + print('Average perplexity:', round(np.mean(perplexities), 4)) + return text + +# Green-yellow-red color scale +def probability_color_scale(prob): + rv = 0 + gv = 0 + if prob <= 0.5: + rv = 'ff' + gv = hex(int(255*prob*2))[2:] + if len(gv) < 2: + gv = '0'*(2 - len(gv)) + gv + else: + rv = hex(int(255 - 255*(prob - 0.5)*2))[2:] + gv = 'ff' + if len(rv) < 2: + rv = '0'*(2 - len(rv)) + rv + return rv + gv + '00' + +# Red component only, white for 0 perplexity (sorry if you're not in dark mode) +def perplexity_color_scale(ppl): + value = hex(max(int(255.0 - params['ppl_scale']*(float(ppl)-1.0)), 0))[2:] + if len(value) < 2: + value = '0'*(2 - len(value)) + value + return 'ff' + value + value + +# Green-yellow-red for probability and blue component for perplexity +def probability_perplexity_color_scale(prob, ppl): + rv = 0 + gv = 0 + bv = hex(min(max(int(params['ppl_scale']*(float(ppl)-1.0)), 0), 255))[2:] + if len(bv) < 2: + bv = '0'*(2 - len(bv)) + bv + if prob <= 0.5: + rv = 'ff' + gv = hex(int(255*prob*2))[2:] + if len(gv) < 2: + gv = '0'*(2 - len(gv)) + gv + else: + rv = hex(int(255 - 255*(prob - 0.5)*2))[2:] + gv = 'ff' + if len(rv) < 2: + rv = '0'*(2 - len(rv)) + rv + return rv + gv + bv + +def add_color_html(token, color): + return f'{token}' + +""" +# This is still very broken at the moment, needs CSS too but I'm not very good at CSS (and neither is GPT-4 apparently) so I still need to figure that out. +def add_dropdown_html(token, color, top_tokens, top_probs): + html = f'{token}
' + for token, prob in zip(top_tokens, top_probs): + # TODO: Background color? Bold for selected token? + # Bigger issue: Why is there a newline after the first token, and the dropdown fails there? + # The HTML ends up like

word

...
, + # even though for all other tokens it shows up correctly. + row_color = probability_color_scale(prob) + html += f'' + html += '
{token}{prob}
' + return html +""" + +def ui(): + color_by_ppl_check = gradio.Checkbox(value=False, label="Color by perplexity", info="Higher perplexity is more red. If also showing probability, higher perplexity has more blue component.") + def update_color_by_ppl_check(x): + params.update({'color_by_perplexity': x}) + color_by_ppl_check.change(update_color_by_ppl_check, color_by_ppl_check, None) + + color_by_prob_check = gradio.Checkbox(value=False, label="Color by probability", info="Green-yellow-red linear scale, with 100% green, 50% yellow, 0% red.") + def update_color_by_prob_check(x): + params.update({'color_by_probability': x}) + color_by_prob_check.change(update_color_by_prob_check, color_by_prob_check, None) + + # Doesn't work yet... + """ + prob_dropdown_check = gradio.Checkbox(value=False, label="Probability dropdown") + def update_prob_dropdown_check(x): + params.update({'probability_dropdown': x}) + prob_dropdown_check.change(update_prob_dropdown_check, prob_dropdown_check, None) + """ From 32f12b8bbf46d69f9ef3ddadfa19ff047a8a54c4 Mon Sep 17 00:00:00 2001 From: jllllll <3887729+jllllll@users.noreply.github.com> Date: Thu, 13 Jul 2023 15:32:37 -0500 Subject: [PATCH 54/54] Bump bitsandbytes Windows wheel to 0.40.0.post4 (#3135) --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 4e012c1e..2ac546ae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,8 +20,8 @@ tensorboard wandb transformers==4.30.2 git+https://github.com/huggingface/peft@03eb378eb914fbee709ff7c86ba5b1d033b89524 -bitsandbytes==0.40.0; platform_system != "Windows" -https://github.com/jllllll/bitsandbytes-windows-webui/releases/download/wheels/bitsandbytes-0.40.0-py3-none-win_amd64.whl; platform_system == "Windows" +bitsandbytes==0.40.0.post4; platform_system != "Windows" +https://github.com/jllllll/bitsandbytes-windows-webui/releases/download/wheels/bitsandbytes-0.40.0.post4-py3-none-win_amd64.whl; platform_system == "Windows" llama-cpp-python==0.1.70; platform_system != "Windows" https://github.com/abetlen/llama-cpp-python/releases/download/v0.1.70/llama_cpp_python-0.1.70-cp310-cp310-win_amd64.whl; platform_system == "Windows" https://github.com/PanQiWei/AutoGPTQ/releases/download/v0.2.2/auto_gptq-0.2.2+cu117-cp310-cp310-win_amd64.whl; platform_system == "Windows"