Create
Import
Export
@@ -2424,6 +2425,66 @@ const SVG_Moveable = ({...props}) => {
${SVG}>
`};
+const SVG_SortIndicator = ({ sortOrder, ...props }) => {
+ return html`
+ <${SVG}
+ ...${props}
+ width="8"
+ height="8"
+ viewBox="0 0 8 8"
+ style=${{ 'margin-left': '2px' }}>
+ ${sortOrder === 'asc'
+ ? html`
`
+ : html`
`}
+ ${SVG}>
+`};
+
+const SVG_SortName = ({...props}) => {
+ return html`
+ <${SVG}
+ ...${props}
+ width="16"
+ height="16"
+ viewBox="0 0 16 16">
+
+
+ ${SVG}>
+`};
+
+const SVG_SortTokens = ({...props}) => {
+ return html`
+ <${SVG}
+ ...${props}
+ width="16"
+ height="16"
+ viewBox="0 0 16 16">
+
+ ${SVG}>
+`};
+
+const SVG_SortSize = ({...props}) => {
+ return html`
+ <${SVG}
+ ...${props}
+ width="16"
+ height="16"
+ viewBox="0 0 16 16">
+
+ ${SVG}>
+`};
+
+const SVG_SortDate = ({...props}) => {
+ return html`
+ <${SVG}
+ ...${props}
+ width="16"
+ height="16"
+ viewBox="0 0 16 16">
+
+ ${SVG}>
+`};
+
+
function Modal({ isOpen, onClose, title, description, children, ...props }) {
if (!isOpen) {
return null;
@@ -3725,6 +3786,423 @@ function InstructModal({ isOpen, closeModal, predict, cancel, modalState, templa
`;
}
+function SavedPromptsModal({ isOpen, closeModal, cancel }) {
+ const [savedPrompts, setSavedPrompts] = useState([]);
+ const [simplePrompts, setSimplePrompts] = useState([]);
+ const [selectedPrompt, setSelectedPrompt] = useState(null);
+ const [loading, setLoading] = useState(false);
+ const [saveName, setSaveName] = useState('');
+ const [loadSlot, setLoadSlot] = useState('');
+ const [activeTab, setActiveTab] = useState('saved'); // 'saved' or 'simple'
+ const [sortBy, setSortBy] = useState('name'); // 'name', 'tokens', 'size', 'date'
+ const [sortOrder, setSortOrder] = useState('asc'); // 'asc' or 'desc'
+ const [listWidth, setListWidth] = useState(300);
+ const [modalHeight, setModalHeight] = useState(500);
+ const [renameNewName, setRenameNewName] = useState('');
+ const [renameOldName, setRenameOldName] = useState('');
+
+ // Fetch data on modal open
+ useEffect(() => {
+ if (isOpen) {
+ fetchSavedPrompts();
+ fetchSimplePrompts();
+ }
+ }, [isOpen]);
+
+ const fetchSavedPrompts = async () => {
+ setLoading(true);
+ try {
+ const res = await fetch('/list');
+ const data = await res.json();
+ setSavedPrompts(data);
+ } catch (error) {
+ console.error('Failed to fetch saved prompts:', error);
+ }
+ setLoading(false);
+ };
+
+ const fetchSimplePrompts = async () => {
+ try {
+ const res = await fetch('/slots/list');
+ const data = await res.json();
+ setSimplePrompts(data);
+ } catch (error) {
+ console.error('Failed to fetch simple prompts:', error);
+ }
+ };
+
+ const handleSort = (type) => {
+ if (sortBy === type) {
+ setSortOrder(sortOrder === 'asc' ? 'desc' : 'asc');
+ } else {
+ setSortBy(type);
+ setSortOrder('asc');
+ }
+ };
+
+ const sortedPrompts = [...savedPrompts].sort((a, b) => {
+ let compareValue = 0;
+ switch (sortBy) {
+ case 'name':
+ compareValue = a.filename.localeCompare(b.filename);
+ break;
+ case 'tokens':
+ compareValue = a.token_count - b.token_count;
+ break;
+ case 'size':
+ compareValue = a.filesize - b.filesize;
+ break;
+ case 'date':
+ compareValue = (new Date(a.mtime))- (new Date(b.mtime));
+ break;
+ }
+ return sortOrder === 'asc' ? compareValue : -compareValue;
+ });
+
+ const handleSavePrompt = async (slot_id) => {
+ if (!saveName.trim()) {
+ alert('Please enter a name for the saved prompt');
+ return;
+ }
+
+ try {
+ await fetch('/slots/' + slot_id + '?action=save', {
+ method: 'POST',
+ body: JSON.stringify({
+ filename: `${saveName}.bin`,
+ })
+ });
+ setSaveName('');
+ fetchSavedPrompts();
+ } catch (error) {
+ console.error('Failed to save prompt:', error);
+ }
+ };
+
+ const handleLoadPrompt = async (filename) => {
+ if (loadSlot === '') {
+ alert('Please select a slot to restore to');
+ return;
+ }
+ if (confirm(`Load "${filename}" into slot "${loadSlot}"? This will overwrite what is currently there.`)) {
+ try {
+ await fetch('/slots/' + loadSlot + '?action=restore', {
+ method: 'POST',
+ body: JSON.stringify({ filename })
+ });
+ fetchSimplePrompts();
+ } catch (error) {
+ console.error('Failed to load prompt:', error);
+ }
+ }
+ };
+
+ const handleDeletePrompt = async (filename) => {
+ if (confirm(`Delete "${filename}"? This cannot be undone.`)) {
+ try {
+ await fetch('/delete_prompt', {
+ method: 'POST',
+ body: JSON.stringify({ filename })
+ });
+ setSelectedPrompt(null);
+ fetchSavedPrompts();
+ } catch (error) {
+ console.error('Failed to delete prompt:', error);
+ }
+ }
+ };
+
+ const handleRenamePrompt = async (old_filename, new_filename) => {
+ try {
+ await fetch('/rename_prompt', {
+ method: 'POST',
+ body: JSON.stringify({ old_filename, new_filename })
+ });
+ fetchSavedPrompts();
+ } catch (error) {
+ console.error('Failed to rename prompt:', error);
+ }
+ };
+
+ const startRenamePrompt = (old_filename) => {
+ setRenameNewName(old_filename);
+ setRenameOldName(old_filename);
+ };
+
+ function handleKeyDown(old_filename, key) {
+ if (event.key === 'Enter') {
+ if (renameOldName !== undefined)
+ handleRenamePrompt(old_filename, renameNewName);
+ } else if (event.key === 'Escape') {
+ if (renameOldName !== undefined)
+ setRenameOldName(undefined);
+ }
+ }
+
+ const SortButton = ({ type, icon }) => html`
+
handleSort(type)}
+ title="Sort by ${type}"
+ style=${{'padding': '4px', 'border': 'none', 'background': 'transparent', 'cursor': 'pointer', 'opacity': sortBy === type ? '1' : '0.5'}}>
+ ${icon}
+ ${sortBy === type && html`<${SVG_SortIndicator} sortOrder=${sortOrder} />`}
+
+ `;
+
+
+ const formatFileSize = (bytes) => {
+ const units = ['B', 'KB', 'MB', 'GB'];
+ let size = bytes;
+ let unitIndex = 0;
+ while (size >= 1024 && unitIndex < units.length - 1) {
+ size /= 1024;
+ unitIndex++;
+ }
+ return `${size.toFixed(1)} ${units[unitIndex]}`;
+ };
+
+return html`
+ <${Modal}
+ isOpen=${isOpen}
+ onClose=${closeModal}
+ title="Saved Prompts"
+ description="Manage your saved prompts and slots."
+ style=${{
+ 'height': `${modalHeight}px`,
+ 'resize': 'vertical',
+ 'overflow': 'auto',
+ 'minHeight': '200px',
+ 'maxHeight': '90vh'
+ }}>
+
+
+
+
+
+
+
+
+ setActiveTab('saved')}
+ style=${{ 'flex': 1 }}>
+ Load Prompts
+
+ setActiveTab('simple')}
+ style=${{ 'flex': 1 }}>
+ Save Slots
+
+
+
+ ${activeTab === 'saved' ? html`
+
+
+ <${SortButton} type="name" icon=${html`<${SVG_SortName}/>`}/>
+ <${SortButton} type="tokens" icon=${html`<${SVG_SortTokens}/>`}/>
+ <${SortButton} type="size" icon=${html`<${SVG_SortSize}/>`}/>
+ <${SortButton} type="date" icon=${html`<${SVG_SortDate}/>`}/>
+
+
+
+ ${loading ? html`
+
+ Loading...
+
+ ` : html`
+
+ `}` : html`
+
+
+ `}
+
+
+
+
{
+ const startX = e.clientX;
+ const startWidth = listWidth;
+
+ const handleMouseMove = (e) => {
+ const newWidth = Math.max(200, Math.min(600, startWidth + e.clientX - startX));
+ setListWidth(newWidth);
+ };
+
+ const handleMouseUp = () => {
+ document.removeEventListener('mousemove', handleMouseMove);
+ document.removeEventListener('mouseup', handleMouseUp);
+ };
+
+ document.addEventListener('mousemove', handleMouseMove);
+ document.addEventListener('mouseup', handleMouseUp);
+ }}>
+
+
+
+
+
+ ${selectedPrompt ? html`
+
+
${Object.hasOwn(selectedPrompt, 'slot_id') ? `Slot #${selectedPrompt.slot_id}` : selectedPrompt.filename.replace('.bin', '')}
+ ${Object.hasOwn(selectedPrompt, 'slot_id') && html`
+
+
${selectedPrompt.token_count.toLocaleString()} tokens
+
+ <${InputBox}
+ 'label'=""
+ 'type'="text"
+ 'placeholder'="Name for saved prompt"
+ 'readOnly'=${!!cancel}
+ 'value'=${saveName}
+ 'onValueChange'=${setSaveName}
+ style=${{ 'flex': 1, 'width': '10vw'}}/>
+
handleSavePrompt(selectedPrompt.slot_id)}
+ style=${{'font-size': '0.9em'}}>
+ Save Prompt
+
+
+ `}
+ ${Object.hasOwn(selectedPrompt, 'filename') && html`
+
+
${selectedPrompt.token_count.toLocaleString()} tokens
+
${formatFileSize(selectedPrompt.filesize)}
+
${new Date(selectedPrompt.mtime).toLocaleString()}
+
+ <${SelectBox}
+ label="Slot"
+ value=${loadSlot}
+ onValueChange=${setLoadSlot}
+ options=${[{ name: '', value: '' }, ...simplePrompts.map(prompt => ({ name: prompt.slot_id, value: prompt.slot_id}))]}/>
+
handleLoadPrompt(selectedPrompt.filename)}
+ style=${{'padding': '2px 8px', 'font-size': '0.9em'}}>
+ Load This Prompt
+
+
+ `}
+
+
+ ` : html`
+
+ Select a saved prompt to preview
+
+ `}
+
+
+ ${Modal}>
+ `;
+}
+
+
function Widget({ isOpen, onClose, title, id, children, ...props }) {
if (!isOpen) {
return null;
@@ -4952,6 +5430,13 @@ export function App({ sessionStorage, templateStorage, useSessionState, useDBTem
const [promptPreviewReroll, setPromptPreviewReroll] = useState(0);
const [showPromptPreview, setShowPromptPreview] = useSessionState('promptPreview', defaultPresets.promptPreview);
const [promptPreviewTokens, setPromptPreviewTokens] = useSessionState('promptPreviewTokens', defaultPresets.promptPreviewTokens);
+ const [zstdTable, setZstdTable] = useState('');
+ const [zstdColumn, setZstdColumn] = useState('');
+ const [zstdLevel, setZstdLevel] = useState(3);
+ const [zstdRatio, setZstdRatio] = useState(0.1);
+ const [showCustomMaintenance, setShowCustomMaintenance] = useState(false);
+ const [maintenanceDuration, setMaintenanceDuration] = useState('');
+ const [maintenanceDbLoad, setMaintenanceDbLoad] = useState(0.5);
function replacePlaceholders(string,placeholders) {
// give placeholders as json object
@@ -6865,6 +7350,118 @@ export function App({ sessionStorage, templateStorage, useSessionState, useDBTem
${CollapsibleGroup}>
+ <${CollapsibleGroup} label="Database Tools" expanded>
+
+ {
+ if (confirm("Optimize database storage? This may temporarily impact performance.")) {
+ await fetch('/vacuum', { method: 'GET' });
+ }
+ }}>
+ VACUUM
+
+ {
+ const res = await fetch('/zstd_get_configs');
+ const data = await res.json();
+ if (data.ok) {
+ console.log(data.configs);
+ }
+ else {
+ console.log(data.message);
+ }
+ }}>
+ Show Configs
+
+
+
+
+ <${InputSlider} label="Compression Level" type="number" step="1" min="1" max="22"
+ readOnly=${!!cancel} value=${zstdLevel} onValueChange=${setZstdLevel}/>
+ <${InputSlider} label="Samples Ratio" type="number" step="0.05" max="1"
+ readOnly=${!!cancel} value=${zstdRatio} onValueChange=${setZstdRatio}/>
+ {
+ await fetch('/zstd_enable_transparent', {
+ method: 'POST',
+ body: JSON.stringify({
+ table: 'sessions',
+ column: 'data',
+ compression_level: zstdLevel,
+ train_dict_samples_ratio: zstdRatio
+ })
+ });
+ }}>
+ Enable
+
+
+
+
+
+
+ {
+ await fetch('/zstd_incremental_maintenance', {
+ method: 'POST',
+ body: JSON.stringify({ duration: null, db_load: 1.0 })
+ });
+ }}>
+ Full Maintenance
+
+ {
+ await fetch('/zstd_incremental_maintenance', {
+ method: 'POST',
+ body: JSON.stringify({ duration: 0, db_load: 1.0 })
+ });
+ }}>
+ Single Item
+
+ setShowCustomMaintenance(!showCustomMaintenance)}>
+ Custom
+
+
+
+ ${showCustomMaintenance && html`
+
+ <${InputBox} label="Duration (sec)" type="number"
+ readOnly=${!!cancel}
+ value=${maintenanceDuration}
+ onValueChange=${setMaintenanceDuration}
+ placeholder="Optional"/>
+ <${InputSlider} label="DB Load" type="number" step="0.1" max="1"
+ readOnly=${!!cancel} value=${maintenanceDbLoad} onValueChange=${setMaintenanceDbLoad}/>
+ {
+ await fetch('/zstd_incremental_maintenance', {
+ method: 'POST',
+ body: JSON.stringify({
+ duration: maintenanceDuration ? parseFloat(maintenanceDuration) : null,
+ db_load: maintenanceDbLoad
+ })
+ });
+ }}>
+ Run
+
+
+ `}
+${CollapsibleGroup}>
+
+ <${CollapsibleGroup} label="Slot and KV Save Mgmt">
+
toggleModal("saved_prompts")}>
+
+ Manage Slots and Saved Prompt
+
+ ${CollapsibleGroup}>
+
${!!tokens && html`
<${InputBox} label="Tokens" value="${tokens}${tokensPerSec ? ` (${tokensPerSec.toFixed(2)} T/s)` : ``}" readOnly/>`}
@@ -7066,6 +7663,11 @@ export function App({ sessionStorage, templateStorage, useSessionState, useDBTem
drySequenceBreakersError=${drySequenceBreakersError}
bannedTokensError=${bannedTokensError}/>
+ <${SavedPromptsModal}
+ isOpen=${modalState.saved_prompts}
+ closeModal=${() => toggleModal("saved_prompts")}
+ cancel=${cancel}/>
+
<${EditorContextMenu}
isOpen=${contextMenuState.visible}
closeMenu=${() => setContextMenuState({ visible: false, x: 0, y: 0 })}
diff --git a/examples/server/server.cpp b/examples/server/server.cpp
index 9c995da7..d44a7dbb 100644
--- a/examples/server/server.cpp
+++ b/examples/server/server.cpp
@@ -3345,7 +3345,8 @@ int main(int argc, char ** argv) {
{ "system_prompt", ctx_server.system_prompt.c_str() },
{ "default_generation_settings", ctx_server.default_generation_settings_for_props },
{ "total_slots", ctx_server.params.n_parallel },
- { "chat_template", curr_tmpl.c_str() }
+ { "chat_template", curr_tmpl.c_str() },
+ { "n_ctx", ctx_server.n_ctx }
};
res.set_content(data.dump(), "application/json; charset=utf-8");
@@ -3760,9 +3761,28 @@ int main(int argc, char ** argv) {
std::vector tokens(n_token_count);
file.read(reinterpret_cast(tokens.data()), tokens.size() * sizeof(llama_token));
+ //C++17 is not modern enough to have a nice and portable way to get the mtime of a file
+ //so the following seems to be needed
+ auto ftime = fs::last_write_time(entry.path());
+ auto system_time = std::chrono::time_point_cast(
+ ftime - fs::file_time_type::clock::now() + std::chrono::system_clock::now()
+ );
+ std::time_t c_time = std::chrono::system_clock::to_time_t(system_time);
+ std::tm tm_struct;
+ #if defined(_WIN32)
+ localtime_s(&tm_struct, &c_time);
+ #else
+ localtime_r(&c_time, &tm_struct);
+ #endif
+ std::ostringstream oss;
+ oss << std::put_time(&tm_struct, "%Y-%m-%d %H:%M:%S");
+ auto str_time = oss.str();
+
+
response.push_back({
{"filename", entry.path().filename().string()},
{"filesize", entry.file_size()},
+ {"mtime", str_time},
{"token_count", n_token_count},
{"prompt", tokens_to_str(ctx_server.ctx, tokens.cbegin(), tokens.cend())}
});
@@ -3774,6 +3794,131 @@ int main(int argc, char ** argv) {
res.set_content(response.dump(), "application/json; charset=utf-8");
};
+ const auto list_slot_prompts = [&ctx_server, ¶ms](const httplib::Request& req, httplib::Response& res) {
+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
+ json response = json::array();
+ for (server_slot & slot : ctx_server.slots) {
+ response.push_back({
+ {"slot_id", slot.id},
+ {"token_count", slot.cache_tokens.size()},
+ {"prompt", tokens_to_str(ctx_server.ctx, slot.cache_tokens.cbegin(), slot.cache_tokens.cend())}
+ });
+ }
+ res.set_content(response.dump(), "application/json; charset=utf-8");
+ };
+
+
+ const auto delete_saved_prompt = [&ctx_server, ¶ms](const httplib::Request& req, httplib::Response& res)-> void {
+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
+ json response;
+ namespace fs = std::filesystem;
+
+ try {
+ const json body = json::parse(req.body);
+ const std::string filename_str = body.at("filename");
+
+ // prevent directory traversal attacks
+ if (filename_str.find("..") != std::string::npos || filename_str.find('/') != std::string::npos || filename_str.find('\\') != std::string::npos) {
+ res.status = 400;
+ response = {{"error", "Invalid filename format."}};
+ res.set_content(response.dump(), "application/json; charset=utf-8");
+ return;
+ }
+
+ const fs::path file_to_delete = fs::path(params.slot_save_path) / fs::path(filename_str);
+
+ if (!fs::exists(file_to_delete) || !fs::is_regular_file(file_to_delete)) {
+ res.status = 404;
+ response = {{"error", "File not found."}};
+ res.set_content(response.dump(), "application/json; charset=utf-8");
+ return;
+ }
+
+ if (fs::remove(file_to_delete)) {
+ response = {
+ {"status", "deleted"},
+ {"filename", filename_str}
+ };
+ } else {
+ res.status = 500;
+ response = {{"error", "Failed to delete the file."}};
+ }
+ } catch (const json::parse_error& e) {
+ res.status = 400;
+ response = {{"error", "Invalid JSON request body."}};
+ } catch (const json::out_of_range& e) {
+ res.status = 400;
+ response = {{"error", "Missing 'filename' key in request body."}};
+ } catch (const std::exception& e) {
+ res.status = 500;
+ response = {{"error", e.what()}};
+ }
+ res.set_content(response.dump(), "application/json; charset=utf-8");
+ };
+
+ const auto rename_saved_prompt = [&ctx_server, ¶ms](const httplib::Request& req, httplib::Response& res)-> void {
+ res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
+ json response;
+ namespace fs = std::filesystem;
+
+ try {
+ const json body = json::parse(req.body);
+ const std::string old_filename_str = body.at("old_filename");
+ const std::string new_filename_str = body.at("new_filename");
+
+ if (old_filename_str.find("..") != std::string::npos || old_filename_str.find_first_of("/\\") != std::string::npos ||
+ new_filename_str.find("..") != std::string::npos || new_filename_str.find_first_of("/\\") != std::string::npos) {
+ res.status = 400;
+ response = {{"error", "Invalid filename format."}};
+ res.set_content(response.dump(), "application/json; charset=utf-8");
+ return;
+ }
+
+ const fs::path old_path = fs::path(params.slot_save_path) / old_filename_str;
+ const fs::path new_path = fs::path(params.slot_save_path) / new_filename_str;
+
+ if (!fs::exists(old_path) || !fs::is_regular_file(old_path)) {
+ res.status = 404;
+ response = {{"error", "Source file not found."}};
+ res.set_content(response.dump(), "application/json; charset=utf-8");
+ return;
+ }
+
+ if (fs::exists(new_path)) {
+ res.status = 409;
+ response = {{"error", "Destination filename already exists."}};
+ res.set_content(response.dump(), "application/json; charset=utf-8");
+ return;
+ }
+
+ std::error_code ec;
+ fs::rename(old_path, new_path, ec);
+
+ if (ec) {
+ res.status = 500;
+ response = {{"error", "Failed to rename file: " + ec.message()}};
+ } else {
+ response = {
+ {"status", "renamed"},
+ {"old_filename", old_filename_str},
+ {"new_filename", new_filename_str}
+ };
+ }
+
+ } catch (const json::parse_error& e) {
+ res.status = 400;
+ response = {{"error", "Invalid JSON request body."}};
+ } catch (const json::out_of_range& e) {
+ res.status = 400;
+ response = {{"error", "Missing 'old_filename' or 'new_filename' in request body."}};
+ } catch (const std::exception& e) {
+ res.status = 500;
+ response = {{"error", e.what()}};
+ }
+
+ res.set_content(response.dump(), "application/json; charset=utf-8");
+ };
+
auto handle_static_file = [](unsigned char * content, size_t len, const char * mime_type) {
return [content, len, mime_type](const httplib::Request &, httplib::Response & res) {
res.set_content(reinterpret_cast(content), len, mime_type);
@@ -3781,7 +3926,6 @@ int main(int argc, char ** argv) {
};
};
-
const auto handle_version = [¶ms, sqlite_extension_loaded](const httplib::Request&, httplib::Response& res) {
res.set_content(
json{{"version", 4},
@@ -3967,10 +4111,14 @@ int main(int argc, char ** argv) {
svr->Post("/lora-adapters", handle_lora_adapters_apply);
// Save & load slots
svr->Get ("/slots", handle_slots);
+ svr->Get ("/slots/list", list_slot_prompts);
if (!params.slot_save_path.empty()) {
// these endpoints rely on slot_save_path existing
svr->Post("/slots/:id_slot", handle_slots_action);
svr->Get ("/list", list_saved_prompts);
+ svr->Post("/delete_prompt", delete_saved_prompt);
+ svr->Post("/rename_prompt", rename_saved_prompt);
+
}
svr->Get ("/version", handle_version);
if (!params.sql_save_file.empty()) {