Merge pull request #5959 from oobabooga/dev

Merge dev branch

css/highlightjs/github-dark.min.css

@@ -0,0 +1,10 @@
+pre code.hljs{display:block;overflow-x:auto;padding:1em}code.hljs{padding:3px 5px}/*!
+  Theme: GitHub Dark
+  Description: Dark theme as seen on github.com
+  Author: github.com
+  Maintainer: @Hirse
+  Updated: 2021-05-15
+
+  Outdated base version: https://github.com/primer/github-syntax-dark
+  Current colors taken from GitHub's CSS
+*/.hljs{color:#c9d1d9;background:#0d1117}.hljs-doctag,.hljs-keyword,.hljs-meta .hljs-keyword,.hljs-template-tag,.hljs-template-variable,.hljs-type,.hljs-variable.language_{color:#ff7b72}.hljs-title,.hljs-title.class_,.hljs-title.class_.inherited__,.hljs-title.function_{color:#d2a8ff}.hljs-attr,.hljs-attribute,.hljs-literal,.hljs-meta,.hljs-number,.hljs-operator,.hljs-selector-attr,.hljs-selector-class,.hljs-selector-id,.hljs-variable{color:#79c0ff}.hljs-meta .hljs-string,.hljs-regexp,.hljs-string{color:#a5d6ff}.hljs-built_in,.hljs-symbol{color:#ffa657}.hljs-code,.hljs-comment,.hljs-formula{color:#8b949e}.hljs-name,.hljs-quote,.hljs-selector-pseudo,.hljs-selector-tag{color:#7ee787}.hljs-subst{color:#c9d1d9}.hljs-section{color:#1f6feb;font-weight:700}.hljs-bullet{color:#f2cc60}.hljs-emphasis{color:#c9d1d9;font-style:italic}.hljs-strong{color:#c9d1d9;font-weight:700}.hljs-addition{color:#aff5b4;background-color:#033a16}.hljs-deletion{color:#ffdcd7;background-color:#67060c}

css/highlightjs/highlightjs-copy.min.css

@@ -0,0 +1 @@
+.hljs-copy-wrapper{position:relative;overflow:hidden}.hljs-copy-wrapper:hover .hljs-copy-button,.hljs-copy-button:focus{transform:translateX(0)}.hljs-copy-button{position:absolute;transform:translateX(calc(100% + 1.125em));top:1em;right:1em;width:2rem;height:2rem;text-indent:-9999px;color:#fff;border-radius:.25rem;border:1px solid #ffffff22;background-color:#2d2b57;background-color:var(--hljs-theme-background);background-image:url('data:image/svg+xml;utf-8,<svg width="16" height="16" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" clip-rule="evenodd" d="M6 5C5.73478 5 5.48043 5.10536 5.29289 5.29289C5.10536 5.48043 5 5.73478 5 6V20C5 20.2652 5.10536 20.5196 5.29289 20.7071C5.48043 20.8946 5.73478 21 6 21H18C18.2652 21 18.5196 20.8946 18.7071 20.7071C18.8946 20.5196 19 20.2652 19 20V6C19 5.73478 18.8946 5.48043 18.7071 5.29289C18.5196 5.10536 18.2652 5 18 5H16C15.4477 5 15 4.55228 15 4C15 3.44772 15.4477 3 16 3H18C18.7956 3 19.5587 3.31607 20.1213 3.87868C20.6839 4.44129 21 5.20435 21 6V20C21 20.7957 20.6839 21.5587 20.1213 22.1213C19.5587 22.6839 18.7957 23 18 23H6C5.20435 23 4.44129 22.6839 3.87868 22.1213C3.31607 21.5587 3 20.7957 3 20V6C3 5.20435 3.31607 4.44129 3.87868 3.87868C4.44129 3.31607 5.20435 3 6 3H8C8.55228 3 9 3.44772 9 4C9 4.55228 8.55228 5 8 5H6Z" fill="white"/><path fill-rule="evenodd" clip-rule="evenodd" d="M7 3C7 1.89543 7.89543 1 9 1H15C16.1046 1 17 1.89543 17 3V5C17 6.10457 16.1046 7 15 7H9C7.89543 7 7 6.10457 7 5V3ZM15 3H9V5H15V3Z" fill="white"/></svg>');background-repeat:no-repeat;background-position:center;transition:background-color 200ms ease,transform 200ms ease-out}.hljs-copy-button:hover{border-color:#ffffff44}.hljs-copy-button:active{border-color:#ffffff66}.hljs-copy-button[data-copied="true"]{text-indent:0;width:auto;background-image:none}@media(prefers-reduced-motion){.hljs-copy-button{transition:none}}.hljs-copy-alert{clip:rect(0 0 0 0);clip-path:inset(50%);height:1px;overflow:hidden;position:absolute;white-space:nowrap;width:1px}

css/html_instruct_style.css

@@ -1,13 +1,12 @@
 .chat {
-    background: var(--block-background-fill);
+    background: transparent;
     padding: 24px 19px;
     padding-right: 19px !important;
     padding-top: 0;
-    border: 1px solid var(--block-border-color);
 }
 
 .chat > .messages {
-    padding-top: 28px !important;
+    padding-top: 18px !important;
 }
 
 .message {
@@ -50,23 +49,24 @@
 
 .gradio-container .chat .assistant-message {
     padding: 20px;
-    background: var(--background-fill-secondary);
-    margin-top: 12px !important;
-    margin-bottom: 24px !important;
-    margin-right: 16px;
-    border-radius: 22px;
-    border-bottom-left-radius: 0;
+    background: var(--color-grey-200);
+    margin-top: 9px !important;
+    margin-bottom: 12px !important;
+    border-radius: 7px;
     border: 1px solid var(--border-color-primary);
 }
 
+.dark .chat .assistant-message {
+    background: var(--color-grey-800);
+}
+
 .gradio-container .chat .user-message {
     padding: 20px;
-    background-color: var(--color-accent-soft);
-    margin-bottom: 12px !important;
-    margin-left: 16px;
-    border-radius: 22px;
+    padding-left: 0px;
+    padding-right: 0px;
+    background-color: transparent;
+    border-radius: 8px;
     border-bottom-right-radius: 0;
-    border: 1px solid var(--border-color-accent-subdued);
 }
 
 .gradio-container .chat .assistant-message:last-child, .gradio-container .chat .user-message:last-child {

css/main.css

@@ -96,7 +96,7 @@ gradio-app > :first-child {
 
 .header_bar {
     background-color: #f7f7f7;
-    box-shadow: 0 2px 3px rgba(22 22 22 / 35%);
+    box-shadow: 0 0px 3px rgba(22 22 22 / 35%);
     margin-bottom: 0;
     overflow-x: scroll;
     margin-left: calc(-1 * var(--size-4));
@@ -220,7 +220,7 @@ button {
 }
 
 .pretty_scrollbar::-webkit-scrollbar {
-    width: 5px;
+    width: 7px;
 }
 
 .pretty_scrollbar::-webkit-scrollbar-track {
@@ -229,12 +229,12 @@ button {
 
 .pretty_scrollbar::-webkit-scrollbar-thumb,
 .pretty_scrollbar::-webkit-scrollbar-thumb:hover {
-    background: #c5c5d2;
+    background: var(--neutral-300);
 }
 
 .dark .pretty_scrollbar::-webkit-scrollbar-thumb,
 .dark .pretty_scrollbar::-webkit-scrollbar-thumb:hover {
-    background: #374151;
+    background: var(--neutral-700);
 }
 
 .pretty_scrollbar::-webkit-resizer {
@@ -433,11 +433,12 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* {
 .message-body code {
     white-space: pre-wrap !important;
     word-wrap: break-word !important;
-    border: 1px solid var(--border-color-primary);
-    border-radius: var(--radius-sm);
-    background: var(--background-fill-secondary);
-    font-size: 90%;
+    border: 1px solid #666666;
+    border-radius: 5px;
+    font-size: 82%;
     padding: 1px 3px;
+    background: #0d1117 !important;
+    color: rgb(201, 209, 217);
 }
 
 .message-body pre > code {
@@ -518,8 +519,12 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* {
     text-overflow: ellipsis;
 }
 
+.dark #show-controls span {
+    color: var(--neutral-400);
+}
+
 #show-controls span {
-    opacity: 0.6;
+    color: var(--neutral-600);
 }
 
 #typing-container {
@@ -590,8 +595,8 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* {
     position: absolute;
     bottom: 80%;
     left: 0;
-    background-color: var(--background-fill-secondary);
-    box-shadow: 0 0 10px rgb(0 0 0 / 50%);
+    background-color: var(--background-fill-primary);
+    box-shadow: 0 0 5px rgb(0 0 0 / 25%);
     z-index: 10000;
     min-width: 330px;
     flex-direction: column;
@@ -601,6 +606,7 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* {
     width: 100%;
     background: transparent !important;
     border-radius: 0 !important;
+    border-color: var(--border-color-primary);
     justify-content: space-between;
     margin: 0 !important;
     height: 36px;
@@ -611,7 +617,7 @@ div.svelte-362y77>*, div.svelte-362y77>.form>* {
 }
 
 .hover-menu button:not(#clear-history-confirm):last-child {
-    border-bottom: var(--button-border-width) solid var(--button-secondary-border-color) !important;
+    border-bottom: var(--button-border-width) solid var(--border-color-primary) !important;
 }
 
 .hover-menu button:hover {

download-model.py

@@ -15,10 +15,12 @@ import os
 import re
 import sys
 from pathlib import Path
+from time import sleep
 
 import requests
 import tqdm
 from requests.adapters import HTTPAdapter
+from requests.exceptions import ConnectionError, RequestException, Timeout
 from tqdm.contrib.concurrent import thread_map
 
 base = os.environ.get("HF_ENDPOINT") or "https://huggingface.co"
@@ -177,50 +179,65 @@ class ModelDownloader:
         return output_folder
 
     def get_single_file(self, url, output_folder, start_from_scratch=False):
-        session = self.get_session()
         filename = Path(url.rsplit('/', 1)[1])
         output_path = output_folder / filename
-        headers = {}
-        mode = 'wb'
-        if output_path.exists() and not start_from_scratch:
 
-            # Check if the file has already been downloaded completely
-            r = session.get(url, stream=True, timeout=10)
-            total_size = int(r.headers.get('content-length', 0))
-            if output_path.stat().st_size >= total_size:
-                return
+        max_retries = 7
+        attempt = 0
+        while attempt < max_retries:
+            attempt += 1
+            session = self.get_session()
+            headers = {}
+            mode = 'wb'
 
-            # Otherwise, resume the download from where it left off
-            headers = {'Range': f'bytes={output_path.stat().st_size}-'}
-            mode = 'ab'
+            if output_path.exists() and not start_from_scratch:
+                # Resume download
+                r = session.get(url, stream=True, timeout=20)
+                total_size = int(r.headers.get('content-length', 0))
+                if output_path.stat().st_size >= total_size:
+                    return
 
-        with session.get(url, stream=True, headers=headers, timeout=10) as r:
-            r.raise_for_status()  # Do not continue the download if the request was unsuccessful
-            total_size = int(r.headers.get('content-length', 0))
-            block_size = 1024 * 1024  # 1MB
+                headers = {'Range': f'bytes={output_path.stat().st_size}-'}
+                mode = 'ab'
 
-            tqdm_kwargs = {
-                'total': total_size,
-                'unit': 'iB',
-                'unit_scale': True,
-                'bar_format': '{l_bar}{bar}| {n_fmt:6}/{total_fmt:6} {rate_fmt:6}'
-            }
+            try:
+                with session.get(url, stream=True, headers=headers, timeout=30) as r:
+                    r.raise_for_status()  # If status is not 2xx, raise an error
+                    total_size = int(r.headers.get('content-length', 0))
+                    block_size = 1024 * 1024  # 1MB
 
-            if 'COLAB_GPU' in os.environ:
-                tqdm_kwargs.update({
-                    'position': 0,
-                    'leave': True
-                })
+                    tqdm_kwargs = {
+                        'total': total_size,
+                        'unit': 'iB',
+                        'unit_scale': True,
+                        'bar_format': '{l_bar}{bar}| {n_fmt}/{total_fmt} {rate_fmt}'
+                    }
 
-            with open(output_path, mode) as f:
-                with tqdm.tqdm(**tqdm_kwargs) as t:
-                    count = 0
-                    for data in r.iter_content(block_size):
-                        t.update(len(data))
-                        f.write(data)
-                        if total_size != 0 and self.progress_bar is not None:
-                            count += len(data)
-                            self.progress_bar(float(count) / float(total_size), f"{filename}")
+                    if 'COLAB_GPU' in os.environ:
+                        tqdm_kwargs.update({
+                            'position': 0,
+                            'leave': True
+                        })
+
+                    with open(output_path, mode) as f:
+                        with tqdm.tqdm(**tqdm_kwargs) as t:
+                            count = 0
+                            for data in r.iter_content(block_size):
+                                f.write(data)
+                                t.update(len(data))
+                                if total_size != 0 and self.progress_bar is not None:
+                                    count += len(data)
+                                    self.progress_bar(float(count) / float(total_size), f"{filename}")
+
+                    break  # Exit loop if successful
+            except (RequestException, ConnectionError, Timeout) as e:
+                print(f"Error downloading {filename}: {e}.")
+                print(f"That was attempt {attempt}/{max_retries}.", end=' ')
+                if attempt < max_retries:
+                    print(f"Retry begins in {2 ** attempt} seconds.")
+                    sleep(2 ** attempt)
+                else:
+                    print("Failed to download after the maximum number of attempts.")
 
     def start_download_threads(self, file_list, output_folder, start_from_scratch=False, threads=4):
         thread_map(lambda url: self.get_single_file(url, output_folder, start_from_scratch=start_from_scratch), file_list, max_workers=threads, disable=True)

js/highlightjs/highlightjs-copy.min.js

@@ -0,0 +1 @@
+class CopyButtonPlugin{constructor(options={}){self.hook=options.hook;self.callback=options.callback;self.lang=options.lang||document.documentElement.lang||"en"}"after:highlightElement"({el,text}){let button=Object.assign(document.createElement("button"),{innerHTML:locales[lang]?.[0]||"Copy",className:"hljs-copy-button"});button.dataset.copied=false;el.parentElement.classList.add("hljs-copy-wrapper");el.parentElement.appendChild(button);el.parentElement.style.setProperty("--hljs-theme-background",window.getComputedStyle(el).backgroundColor);button.onclick=function(){if(!navigator.clipboard)return;let newText=text;if(hook&&typeof hook==="function"){newText=hook(text,el)||text}navigator.clipboard.writeText(newText).then(function(){button.innerHTML=locales[lang]?.[1]||"Copied!";button.dataset.copied=true;let alert=Object.assign(document.createElement("div"),{role:"status",className:"hljs-copy-alert",innerHTML:locales[lang]?.[2]||"Copied to clipboard"});el.parentElement.appendChild(alert);setTimeout(()=>{button.innerHTML=locales[lang]?.[0]||"Copy";button.dataset.copied=false;el.parentElement.removeChild(alert);alert=null},2e3)}).then(function(){if(typeof callback==="function")return callback(newText,el)})}}}if(typeof module!="undefined"){module.exports=CopyButtonPlugin}const locales={en:["Copy","Copied!","Copied to clipboard"],es:["Copiar","¡Copiado!","Copiado al portapapeles"],fr:["Copier","Copié !","Copié dans le presse-papier"],de:["Kopieren","Kopiert!","In die Zwischenablage kopiert"],ja:["コピー","コピーしました！","クリップボードにコピーしました"],ko:["복사","복사됨!","클립보드에 복사됨"],ru:["Копировать","Скопировано!","Скопировано в буфер обмена"],zh:["复制","已复制!","已复制到剪贴板"],"zh-tw":["複製","已複製!","已複製到剪貼簿"]};

js/katex/auto-render.min.js

@@ -0,0 +1 @@
+!function(e,t){"object"==typeof exports&&"object"==typeof module?module.exports=t(require("katex")):"function"==typeof define&&define.amd?define(["katex"],t):"object"==typeof exports?exports.renderMathInElement=t(require("katex")):e.renderMathInElement=t(e.katex)}("undefined"!=typeof self?self:this,(function(e){return function(){"use strict";var t={771:function(t){t.exports=e}},n={};function r(e){var o=n[e];if(void 0!==o)return o.exports;var i=n[e]={exports:{}};return t[e](i,i.exports,r),i.exports}r.n=function(e){var t=e&&e.__esModule?function(){return e.default}:function(){return e};return r.d(t,{a:t}),t},r.d=function(e,t){for(var n in t)r.o(t,n)&&!r.o(e,n)&&Object.defineProperty(e,n,{enumerable:!0,get:t[n]})},r.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)};var o={};return function(){r.d(o,{default:function(){return d}});var e=r(771),t=r.n(e);const n=function(e,t,n){let r=n,o=0;const i=e.length;for(;r<t.length;){const n=t[r];if(o<=0&&t.slice(r,r+i)===e)return r;"\\"===n?r++:"{"===n?o++:"}"===n&&o--,r++}return-1},i=/^\\begin{/;var a=function(e,t){let r;const o=[],a=new RegExp("("+t.map((e=>e.left.replace(/[-/\\^$*+?.()|[\]{}]/g,"\\$&"))).join("|")+")");for(;r=e.search(a),-1!==r;){r>0&&(o.push({type:"text",data:e.slice(0,r)}),e=e.slice(r));const a=t.findIndex((t=>e.startsWith(t.left)));if(r=n(t[a].right,e,t[a].left.length),-1===r)break;const l=e.slice(0,r+t[a].right.length),s=i.test(l)?l:e.slice(t[a].left.length,r);o.push({type:"math",data:s,rawData:l,display:t[a].display}),e=e.slice(r+t[a].right.length)}return""!==e&&o.push({type:"text",data:e}),o};const l=function(e,n){const r=a(e,n.delimiters);if(1===r.length&&"text"===r[0].type)return null;const o=document.createDocumentFragment();for(let e=0;e<r.length;e++)if("text"===r[e].type)o.appendChild(document.createTextNode(r[e].data));else{const i=document.createElement("span");let a=r[e].data;n.displayMode=r[e].display;try{n.preProcess&&(a=n.preProcess(a)),t().render(a,i,n)}catch(i){if(!(i instanceof t().ParseError))throw i;n.errorCallback("KaTeX auto-render: Failed to parse `"+r[e].data+"` with ",i),o.appendChild(document.createTextNode(r[e].rawData));continue}o.appendChild(i)}return o},s=function(e,t){for(let n=0;n<e.childNodes.length;n++){const r=e.childNodes[n];if(3===r.nodeType){let o=r.textContent,i=r.nextSibling,a=0;for(;i&&i.nodeType===Node.TEXT_NODE;)o+=i.textContent,i=i.nextSibling,a++;const s=l(o,t);if(s){for(let e=0;e<a;e++)r.nextSibling.remove();n+=s.childNodes.length-1,e.replaceChild(s,r)}else n+=a}else if(1===r.nodeType){const e=" "+r.className+" ";-1===t.ignoredTags.indexOf(r.nodeName.toLowerCase())&&t.ignoredClasses.every((t=>-1===e.indexOf(" "+t+" ")))&&s(r,t)}}};var d=function(e,t){if(!e)throw new Error("No element provided to render");const n={};for(const e in t)t.hasOwnProperty(e)&&(n[e]=t[e]);n.delimiters=n.delimiters||[{left:"$$",right:"$$",display:!0},{left:"\\(",right:"\\)",display:!1},{left:"\\begin{equation}",right:"\\end{equation}",display:!0},{left:"\\begin{align}",right:"\\end{align}",display:!0},{left:"\\begin{alignat}",right:"\\end{alignat}",display:!0},{left:"\\begin{gather}",right:"\\end{gather}",display:!0},{left:"\\begin{CD}",right:"\\end{CD}",display:!0},{left:"\\[",right:"\\]",display:!0}],n.ignoredTags=n.ignoredTags||["script","noscript","style","textarea","pre","code","option"],n.ignoredClasses=n.ignoredClasses||[],n.errorCallback=n.errorCallback||console.error,n.macros=n.macros||{},s(e,n)}}(),o=o.default}()}));

js/main.js

@@ -137,6 +137,9 @@ targetElement.addEventListener("scroll", function() {
   } else {
     isScrolled = true;
   }
+
+  doSyntaxHighlighting();
+
 });
 
 // Create a MutationObserver instance
@@ -144,10 +147,6 @@ const observer = new MutationObserver(function(mutations) {
   mutations.forEach(function(mutation) {
     updateCssProperties();
 
-    if(!isScrolled) {
-      targetElement.scrollTop = targetElement.scrollHeight;
-    }
-
     const firstChild = targetElement.children[0];
     if (firstChild.classList.contains("generating")) {
       typing.parentNode.classList.add("visible-dots");
@@ -159,7 +158,13 @@ const observer = new MutationObserver(function(mutations) {
       document.getElementById("Generate").style.display = "flex";
     }
 
+    doSyntaxHighlighting();
   });
+
+  if(!isScrolled) {
+    targetElement.scrollTop = targetElement.scrollHeight;
+  }
+
 });
 
 // Configure the observer to watch for changes in the subtree and attributes
@@ -174,6 +179,66 @@ const config = {
 // Start observing the target element
 observer.observe(targetElement, config);
 
+//------------------------------------------------
+// Handle syntax highlighting / LaTeX
+//------------------------------------------------
+function isElementVisibleOnScreen(element) {
+  const rect = element.getBoundingClientRect();
+  return (
+    rect.left < window.innerWidth &&
+    rect.right > 0 &&
+    rect.top < window.innerHeight &&
+    rect.bottom > 0
+  );
+}
+
+function getVisibleMessagesIndexes() {
+  const elements = document.querySelectorAll(".message-body");
+  const visibleIndexes = [];
+
+  elements.forEach((element, index) => {
+    if (isElementVisibleOnScreen(element) && !element.hasAttribute("data-highlighted")) {
+      visibleIndexes.push(index);
+    }
+  });
+
+  return visibleIndexes;
+}
+
+function doSyntaxHighlighting() {
+  const indexes = getVisibleMessagesIndexes();
+  const elements = document.querySelectorAll(".message-body");
+
+  if (indexes.length > 0) {
+    observer.disconnect();
+
+    indexes.forEach((index) => {
+      const element = elements[index];
+
+      // Perform syntax highlighting
+      const codeBlocks = element.querySelectorAll("pre code");
+
+      codeBlocks.forEach((codeBlock) => {
+        hljs.highlightElement(codeBlock);
+      });
+
+      renderMathInElement(element, {
+        delimiters: [
+          { left: "$$", right: "$$", display: true },
+          { left: "$", right: "$", display: false },
+          { left: "\\(", right: "\\)", display: false },
+          { left: "\\[", right: "\\]", display: true },
+        ],
+      });
+
+      // Tag this element to indicate it has been syntax highlighted
+      element.setAttribute("data-highlighted", "true");
+    });
+
+    observer.observe(targetElement, config);
+  }
+}
+
 //------------------------------------------------
 // Add some scrollbars
 //------------------------------------------------
@@ -470,7 +535,7 @@ respondToRenameVisibility(renameTextArea, handleVisibilityChange);
 // is present at the bottom
 //------------------------------------------------
 
-if (document.getElementById('extensions') === null) {
+if (document.getElementById("extensions") === null) {
   document.getElementById("chat-tab").style.marginBottom = "-29px";
 }
 
@@ -478,8 +543,8 @@ if (document.getElementById('extensions') === null) {
 // Focus on the chat input after starting a new chat
 //------------------------------------------------
 
-document.querySelectorAll('.focus-on-chat-input').forEach(element => {
-  element.addEventListener('click', function() {
-      document.querySelector('#chat-input textarea').focus();
+document.querySelectorAll(".focus-on-chat-input").forEach(element => {
+  element.addEventListener("click", function() {
+    document.querySelector("#chat-input textarea").focus();
   });
 });

models/config.yaml

@@ -194,3 +194,13 @@
   instruction_template: 'ChatML'
 .*command-r:
   instruction_template: 'Command-R'
+.*xwin-lm-70b-v0.1:
+  instruction_template: 'Vicuna-v1.1'
+.*platypus-yi-34b:
+  instruction_template: 'Vicuna-v1.1'
+.*CausalLM-RP-34B:
+  instruction_template: 'ChatML'
+34b-beta:
+  instruction_template: 'ChatML'
+.*airoboros-3_1-yi-34b-200k:
+  instruction_template: 'Llama-v2'

modules/block_requests.py

@@ -45,6 +45,15 @@ def my_open(*args, **kwargs):
 
         file_contents = file_contents.replace(b'\t\t<script\n\t\t\tsrc="https://cdnjs.cloudflare.com/ajax/libs/iframe-resizer/4.3.9/iframeResizer.contentWindow.min.js"\n\t\t\tasync\n\t\t></script>', b'')
         file_contents = file_contents.replace(b'cdnjs.cloudflare.com', b'127.0.0.1')
+        file_contents = file_contents.replace(
+            b'</head>',
+            b'\n    <script src="file/js/katex/katex.min.js"></script>'
+            b'\n    <script src="file/js/katex/auto-render.min.js"></script>'
+            b'\n    <script src="file/js/highlightjs/highlight.min.js"></script>'
+            b'\n    <script src="file/js/highlightjs/highlightjs-copy.min.js"></script>'
+            b'\n    <script>hljs.addPlugin(new CopyButtonPlugin());</script>'
+            b'\n  </head>'
+        )
 
         return io.BytesIO(file_contents)
     else:

modules/html_generator.py

@@ -57,6 +57,14 @@ def convert_to_markdown(string):
     # Code
     string = string.replace('\\begin{code}', '```')
     string = string.replace('\\end{code}', '```')
+    string = string.replace('\\begin{align*}', '$$')
+    string = string.replace('\\end{align*}', '$$')
+    string = string.replace('\\begin{align}', '$$')
+    string = string.replace('\\end{align}', '$$')
+    string = string.replace('\\begin{equation}', '$$')
+    string = string.replace('\\end{equation}', '$$')
+    string = string.replace('\\begin{equation*}', '$$')
+    string = string.replace('\\end{equation*}', '$$')
     string = re.sub(r"(.)```", r"\1\n```", string)
 
     result = ''

modules/models.py

@@ -205,6 +205,7 @@ def huggingface_loader(model_name):
                     'bnb_4bit_compute_dtype': eval("torch.{}".format(shared.args.compute_dtype)) if shared.args.compute_dtype in ["bfloat16", "float16", "float32"] else None,
                     'bnb_4bit_quant_type': shared.args.quant_type,
                     'bnb_4bit_use_double_quant': shared.args.use_double_quant,
+                    'llm_int8_enable_fp32_cpu_offload': True
                 }
 
                 params['quantization_config'] = BitsAndBytesConfig(**quantization_config_params)

modules/shared.py

@@ -46,6 +46,7 @@ settings = {
     'truncation_length_min': 0,
     'truncation_length_max': 200000,
     'max_tokens_second': 0,
+    'max_updates_second': 0,
     'prompt_lookup_num_tokens': 0,
     'custom_stopping_strings': '',
     'custom_token_bans': '',

modules/text_generation.py

@@ -3,7 +3,6 @@ import copy
 import html
 import pprint
 import random
-import re
 import time
 import traceback
 
@@ -85,6 +84,10 @@ def _generate_reply(question, state, stopping_strings=None, is_chat=False, escap
         state = copy.deepcopy(state)
         state['stream'] = True
 
+    min_update_interval = 0
+    if state.get('max_updates_second', 0) > 0:
+        min_update_interval = 1 / state['max_updates_second']
+
     # Generate
     for reply in generate_func(question, original_question, seed, state, stopping_strings, is_chat=is_chat):
         reply, stop_found = apply_stopping_strings(reply, all_stop_strings)
@@ -102,7 +105,14 @@ def _generate_reply(question, state, stopping_strings=None, is_chat=False, escap
 
                 last_update = time.time()
                 yield reply
+
+            # Limit updates to avoid lag in the Gradio UI
+            # API updates are not limited
             else:
+                if cur_time - last_update > min_update_interval:
+                    last_update = cur_time
+                    yield reply
+
                 yield reply
 
         if stop_found or (state['max_tokens_second'] > 0 and shared.stop_everything):
@@ -196,20 +206,6 @@ def formatted_outputs(reply, model_name):
     return html.unescape(reply), generate_basic_html(reply)
 
 
-def fix_galactica(s):
-    """
-    Fix the LaTeX equations in GALACTICA
-    """
-    s = s.replace(r'\[', r'$')
-    s = s.replace(r'\]', r'$')
-    s = s.replace(r'\(', r'$')
-    s = s.replace(r'\)', r'$')
-    s = s.replace(r'$$', r'$')
-    s = re.sub(r'\n', r'\n\n', s)
-    s = re.sub(r"\n{3,}", "\n\n", s)
-    return s
-
-
 def set_manual_seed(seed):
     seed = int(seed)
     if seed == -1:

modules/ui.py

@@ -13,6 +13,12 @@ with open(Path(__file__).resolve().parent / '../css/NotoSans/stylesheet.css', 'r
     css = f.read()
 with open(Path(__file__).resolve().parent / '../css/main.css', 'r') as f:
     css += f.read()
+with open(Path(__file__).resolve().parent / '../css/katex/katex.min.css', 'r') as f:
+    css += f.read()
+with open(Path(__file__).resolve().parent / '../css/highlightjs/github-dark.min.css', 'r') as f:
+    css += f.read()
+with open(Path(__file__).resolve().parent / '../css/highlightjs/highlightjs-copy.min.css', 'r') as f:
+    css += f.read()
 with open(Path(__file__).resolve().parent / '../js/main.js', 'r') as f:
     js = f.read()
 with open(Path(__file__).resolve().parent / '../js/save_files.js', 'r') as f:
@@ -36,7 +42,7 @@ theme = gr.themes.Default(
     button_large_padding='6px 12px',
     body_text_color_subdued='#484848',
     background_fill_secondary='#eaeaea',
-    background_fill_primary='#fafafa',
+    background_fill_primary='var(--neutral-50)',
 )
 
 if Path("notification.mp3").exists():
@@ -116,6 +122,7 @@ def list_interface_input_elements():
         'max_new_tokens',
         'auto_max_new_tokens',
         'max_tokens_second',
+        'max_updates_second',
         'prompt_lookup_num_tokens',
         'seed',
         'temperature',

modules/ui_parameters.py

@@ -85,6 +85,7 @@ def create_ui(default_preset):
                             shared.gradio['truncation_length'] = gr.Slider(value=get_truncation_length(), minimum=shared.settings['truncation_length_min'], maximum=shared.settings['truncation_length_max'], step=256, label='Truncate the prompt up to this length', info='The leftmost tokens are removed if the prompt exceeds this length. Most models require this to be at most 2048.')
                             shared.gradio['prompt_lookup_num_tokens'] = gr.Slider(value=shared.settings['prompt_lookup_num_tokens'], minimum=0, maximum=10, step=1, label='prompt_lookup_num_tokens', info='Activates Prompt Lookup Decoding.')
                             shared.gradio['max_tokens_second'] = gr.Slider(value=shared.settings['max_tokens_second'], minimum=0, maximum=20, step=1, label='Maximum tokens/second', info='To make text readable in real time.')
+                            shared.gradio['max_updates_second'] = gr.Slider(value=shared.settings['max_updates_second'], minimum=0, maximum=24, step=1, label='Maximum UI updates/second', info='Set this if you experience lag in the UI during streaming.')
                             shared.gradio['seed'] = gr.Number(value=shared.settings['seed'], label='Seed (-1 for random)')
                             shared.gradio['skip_special_tokens'] = gr.Checkbox(value=shared.settings['skip_special_tokens'], label='Skip special tokens', info='Some specific models need this unset.')
                             shared.gradio['stream'] = gr.Checkbox(value=shared.settings['stream'], label='Activate text streaming')

settings-template.yaml

@@ -15,6 +15,7 @@ truncation_length: 2048
 truncation_length_min: 0
 truncation_length_max: 200000
 max_tokens_second: 0
+max_updates_second: 0
 prompt_lookup_num_tokens: 0
 custom_stopping_strings: ''
 custom_token_bans: ''