text-generation-webui/modules/RWKV.py

149 lines
5.4 KiB
Python
Raw Normal View History

import copy
2023-02-28 04:09:11 +01:00
import os
2023-02-28 03:50:16 +01:00
from pathlib import Path
2023-02-28 04:09:11 +01:00
2023-02-28 03:50:16 +01:00
import numpy as np
2023-03-06 12:45:49 +01:00
from tokenizers import Tokenizer
2023-02-28 04:09:11 +01:00
import modules.shared as shared
2023-03-08 06:50:49 +01:00
from modules.callbacks import Iteratorize
2023-02-28 04:09:11 +01:00
2023-02-28 03:50:16 +01:00
np.set_printoptions(precision=4, suppress=True, linewidth=200)
os.environ['RWKV_JIT_ON'] = '1'
os.environ["RWKV_CUDA_ON"] = '1' if shared.args.rwkv_cuda_on else '0' # use CUDA kernel for seq mode (much faster)
2023-02-28 03:50:16 +01:00
from rwkv.model import RWKV
from rwkv.utils import PIPELINE, PIPELINE_ARGS
2023-03-01 16:18:17 +01:00
2023-03-01 16:08:55 +01:00
class RWKVModel:
def __init__(self):
pass
2023-02-28 03:50:16 +01:00
2023-03-01 16:08:55 +01:00
@classmethod
def from_pretrained(self, path, dtype="fp16", device="cuda"):
tokenizer_path = Path(f"{path.parent}/20B_tokenizer.json")
2023-03-02 00:02:48 +01:00
if shared.args.rwkv_strategy is None:
model = RWKV(model=str(path), strategy=f'{device} {dtype}')
2023-03-02 00:02:48 +01:00
else:
model = RWKV(model=str(path), strategy=shared.args.rwkv_strategy)
2023-02-28 03:50:16 +01:00
2023-05-10 03:49:39 +02:00
pipeline = PIPELINE(model, str(tokenizer_path))
2023-03-01 16:08:55 +01:00
result = self()
2023-03-01 16:33:09 +01:00
result.pipeline = pipeline
result.model = model
result.cached_context = ""
result.cached_model_state = None
result.cached_output_logits = None
2023-03-01 16:08:55 +01:00
return result
2023-06-17 01:35:38 +02:00
def generate(self, prompt, state, callback=None):
2023-03-01 16:16:11 +01:00
args = PIPELINE_ARGS(
2023-06-17 01:35:38 +02:00
temperature=state['temperature'],
top_p=state['top_p'],
top_k=state['top_k'],
alpha_frequency=0.1, # Frequency Penalty (as in GPT-3)
alpha_presence=0.1, # Presence Penalty (as in GPT-3)
token_ban=[0], # ban the generation of some tokens
token_stop=[]
2023-03-01 16:16:11 +01:00
)
if self.cached_context != "":
2023-06-17 01:35:38 +02:00
if prompt.startswith(self.cached_context):
prompt = prompt[len(self.cached_context):]
else:
self.cached_context = ""
self.cached_model_state = None
self.cached_output_logits = None
2023-06-17 01:35:38 +02:00
# out = self.pipeline.generate(prompt, token_count=state['max_new_tokens'], args=args, callback=callback)
out = self.generate_from_cached_state(prompt, token_count=state['max_new_tokens'], args=args, callback=callback)
return out
2023-03-06 12:45:49 +01:00
2023-06-17 01:35:38 +02:00
def generate_with_streaming(self, *args, **kwargs):
with Iteratorize(self.generate, args, kwargs, callback=None) as generator:
reply = ''
for token in generator:
reply += token
yield reply
2023-03-07 22:17:56 +01:00
# Similar to the PIPELINE.generate, but lets us maintain the cached_model_state
def generate_from_cached_state(self, ctx="", token_count=20, args=None, callback=None):
all_tokens = []
out_str = ''
occurrence = {}
state = copy.deepcopy(self.cached_model_state) if self.cached_model_state is not None else None
# if we ended up with an empty context, just reuse the cached logits
# this can happen if a user undoes a message and then sends the exact message again
# in that case the full context ends up being the same as the cached_context, so the remaining context is empty.
if ctx == "":
out = self.cached_output_logits
2023-06-17 01:35:38 +02:00
token = None
for i in range(token_count):
# forward
tokens = self.pipeline.encode(ctx) if i == 0 else [token]
while len(tokens) > 0:
out, state = self.model.forward(tokens[:args.chunk_len], state)
tokens = tokens[args.chunk_len:]
if i == 0:
begin_token = len(all_tokens)
last_token_posi = begin_token
# cache the model state after scanning the context
2023-05-10 03:49:39 +02:00
# we don't cache the state after processing our own generated tokens because
# the output string might be post-processed arbitrarily. Therefore, what's fed into the model
# on the next round of chat might be slightly different what what it output on the previous round
if i == 0:
self.cached_context += ctx
self.cached_model_state = copy.deepcopy(state)
self.cached_output_logits = copy.deepcopy(out)
2023-05-10 03:49:39 +02:00
# adjust probabilities
for n in args.token_ban:
out[n] = -float('inf')
2023-05-10 03:49:39 +02:00
for n in occurrence:
out[n] -= (args.alpha_presence + occurrence[n] * args.alpha_frequency)
2023-05-10 03:49:39 +02:00
# sampler
token = self.pipeline.sample_logits(out, temperature=args.temperature, top_p=args.top_p, top_k=args.top_k)
if token in args.token_stop:
break
2023-05-10 03:49:39 +02:00
all_tokens += [token]
if token not in occurrence:
occurrence[token] = 1
else:
occurrence[token] += 1
2023-05-10 03:49:39 +02:00
# output
tmp = self.pipeline.decode(all_tokens[last_token_posi:])
2023-05-10 03:49:39 +02:00
if '\ufffd' not in tmp: # is valid utf-8 string?
if callback:
callback(tmp)
out_str += tmp
last_token_posi = begin_token + i + 1
return out_str
2023-03-06 12:45:49 +01:00
class RWKVTokenizer:
def __init__(self):
pass
@classmethod
def from_pretrained(self, path):
tokenizer_path = path / "20B_tokenizer.json"
tokenizer = Tokenizer.from_file(str(tokenizer_path))
2023-03-06 12:45:49 +01:00
result = self()
result.tokenizer = tokenizer
return result
def encode(self, prompt):
return self.tokenizer.encode(prompt).ids
def decode(self, ids):
return self.tokenizer.decode(ids)