turboderp-org/exllamav3
1
0
Fork 0
mirror of https://github.com/turboderp-org/exllamav3.git synced 2026-04-20 14:29:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

chat.py: Limit frequency of markdown renders

Browse Source
This commit is contained in:
turboderp
2026-03-03 22:46:32 +01:00
parent eb1686a840
commit a6cf34574b
2 changed files with 15 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
examples/chat.py
View File

@@ -246,7 +246,7 @@ def main(args):
ctx_exceeded = False
with (
KeyReader() as keyreader,
streamer_cm(args, bot_name, tt[0], tt[1]) as s
streamer_cm(args, bot_name, tt[0], tt[1], args.updates_per_second) as s
):
if prefix:
s.stream(prefix)
@@ -334,5 +334,6 @@ if __name__ == "__main__":
parser.add_argument("-save", "--save", type = str, help = "Save output to file (use with --prompt)")
parser.add_argument("-save_svg", "--save_svg", action = "store_true", help = "Extract SVG from response (use with --save)")
parser.add_argument("-dbg", "--debug", action = "store_true", help = "Print extra debug stuff")
parser.add_argument("-ups", "--updates-per-second", type = int, help = "Max number of console updates per second (markdown console), default: 30", default = 30)
_args = parser.parse_args()
main(_args)

17
examples/chat_console.py
View File

@@ -47,10 +47,12 @@ def read_input_ptk(args, user_name, multiline: bool, prefix: str = None):
class Streamer_basic:
def __init__(self, args, bot_name, think_tag, end_think_tag):
def __init__(self, args, bot_name, think_tag, end_think_tag, updates_per_second):
self.all_text = ""
self.args = args
self.bot_name = bot_name
self.updates_per_second = updates_per_second
def __enter__(self):
print()
@@ -125,7 +127,7 @@ class MarkdownConsoleStream:
return i
class Streamer_rich:
def __init__(self, args, bot_name, think_tag, end_think_tag):
def __init__(self, args, bot_name, think_tag, end_think_tag, updates_per_second):
self.all_text = ""
self.think_text = ""
self.bot_name = bot_name
@@ -135,6 +137,8 @@ class Streamer_rich:
self.is_live = False
self.think_tag = think_tag
self.end_think_tag = end_think_tag
self.updates_per_second = updates_per_second
self.last_update = time.time()
def begin(self):
self.live = MarkdownConsoleStream()
@@ -152,10 +156,11 @@ class Streamer_rich:
return self
def __exit__(self, exc_type, exc_value, traceback):
self.stream("", True)
if self.is_live:
self.live.__exit__(exc_type, exc_value, traceback)
def stream(self, text: str):
def stream(self, text: str, force: bool = False):
if self.args.think and self.think_tag is not None and not self.is_live:
print_text = text
if not self.think_text:
@@ -180,7 +185,11 @@ class Streamer_rich:
if self.think_tag is not None:
formatted_text = formatted_text.replace(self.think_tag, f"`{self.think_tag}`")
formatted_text = formatted_text.replace(self.end_think_tag, f"`{self.end_think_tag}`")
self.live.update(formatted_text)
now = time.time()
if now - self.last_update > 1.0 / self.updates_per_second or force:
self.last_update = now
self.live.update(formatted_text)
class KeyReader:
"""