From bf70c19603627a5bbf6b5a2370bddc9ac45d81db Mon Sep 17 00:00:00 2001 From: cal066 <60696996+cal066@users.noreply.github.com> Date: Sun, 13 Aug 2023 03:04:03 +0000 Subject: [PATCH] ctransformers: move thread and seed parameters (#3543) --- modules/ctransformers_model.py | 7 +++---- modules/loaders.py | 3 +-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/modules/ctransformers_model.py b/modules/ctransformers_model.py index c5bc701a..f5641616 100644 --- a/modules/ctransformers_model.py +++ b/modules/ctransformers_model.py @@ -13,14 +13,12 @@ class CtransformersModel: def from_pretrained(self, path): result = self() - # ctransformers uses -1 for random seed config = AutoConfig.from_pretrained( str(path), threads=shared.args.threads, gpu_layers=shared.args.n_gpu_layers, batch_size=shared.args.n_batch, - stream=True, - seed=(-1 if shared.args.llama_cpp_seed == 0 else shared.args.llama_cpp_seed) + stream=True ) self.model = AutoModelForCausalLM.from_pretrained( @@ -49,6 +47,7 @@ class CtransformersModel: def generate(self, prompt, state, callback=None): prompt = prompt if type(prompt) is str else prompt.decode() + # ctransformers uses -1 for random seed generator = self.model._stream( prompt=prompt, max_new_tokens=state['max_new_tokens'], @@ -57,7 +56,7 @@ class CtransformersModel: top_k=state['top_k'], repetition_penalty=state['repetition_penalty'], last_n_tokens=state['repetition_penalty_range'], - threads=shared.args.threads + seed=state['seed'] ) output = "" diff --git a/modules/loaders.py b/modules/loaders.py index 9a222a72..2b3a50b3 100644 --- a/modules/loaders.py +++ b/modules/loaders.py @@ -95,8 +95,7 @@ loaders_and_params = OrderedDict({ 'n_gpu_layers', 'n_batch', 'threads', - 'model_type', - 'llama_cpp_seed', + 'model_type' ] })