text-generation-webui/modules/logits.py

32 lines
993 B
Python
Raw Normal View History

2023-08-21 01:49:21 +02:00
import torch
from modules import sampler_hijack, shared
from modules.text_generation import generate_reply
2023-08-21 01:49:21 +02:00
global_scores = None
2023-08-21 01:49:21 +02:00
def get_next_logits(prompt, state, use_samplers, previous):
if use_samplers:
state['max_new_tokens'] = 1
state['auto_max_new_tokens'] = False
for _ in generate_reply(prompt, state):
pass
scores = sampler_hijack.global_scores[-1]
else:
tokens = shared.tokenizer.encode(prompt, return_tensors='pt').cuda()
output = shared.model(input_ids=tokens)
scores = output['logits'][-1][-1]
2023-08-21 01:49:21 +02:00
probs = torch.softmax(scores, dim=-1, dtype=torch.float)
2023-08-23 05:35:12 +02:00
topk_values, topk_indices = torch.topk(probs, k=25, largest=True, sorted=True)
topk_values = [f"{float(i):.5f}" for i in topk_values]
2023-08-23 05:35:12 +02:00
tokens = [shared.tokenizer.decode(i) for i in topk_indices]
2023-08-21 01:49:21 +02:00
output = ''
2023-08-23 05:35:12 +02:00
for row in list(zip(topk_values, tokens)):
output += f"{row[0]} - {row[1]}\n"
2023-08-21 01:49:21 +02:00
return output, previous