[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[elpa] externals/elisa f312d189f2 51/98: Fix one word lines in webpage q
From: |
ELPA Syncer |
Subject: |
[elpa] externals/elisa f312d189f2 51/98: Fix one word lines in webpage quotes parsed asyncronously |
Date: |
Wed, 17 Jul 2024 18:58:03 -0400 (EDT) |
branch: externals/elisa
commit f312d189f290eb1fa22f4717da046e13ffd5b855
Author: Sergey Kostyaev <kostyaev.sergey2@wb.ru>
Commit: Sergey Kostyaev <kostyaev.sergey2@wb.ru>
Fix one word lines in webpage quotes parsed asyncronously
---
elisa.el | 66 ++++++++++++++++++++++++++++++++++------------------------------
1 file changed, 35 insertions(+), 31 deletions(-)
diff --git a/elisa.el b/elisa.el
index 40f2de7fd0..e14b62be60 100644
--- a/elisa.el
+++ b/elisa.el
@@ -458,36 +458,36 @@ ARGS contains keys for fine control.
:threshold-amount K -- K is a breakpoint threshold amount.
than T, it will be packed into single semantic chunk."
- (let* ((func (or (plist-get args :function) elisa-semantic-split-function))
- (k (or (plist-get args :threshold-amount)
elisa-breakpoint-threshold-amount))
- (chunks (funcall func))
- (embeddings (cl-remove-if
- #'not
- (mapcar (lambda (s)
- (when (length> (string-trim s) 0)
- (llm-embedding elisa-embeddings-provider s)))
- chunks)))
- (distances (elisa--distances embeddings))
- (threshold (elisa-calculate-threshold k distances))
- (result nil)
- (current (car chunks))
- (tail (cdr chunks)))
- (mapc
- (lambda (el)
- (if (<= el threshold)
- (setq current (concat current (car tail)))
- (push current result)
- (setq current (car tail)))
- (setq tail (cdr tail)))
- distances)
- (push current result)
- (cl-remove-if
- #'string-empty-p
- (mapcar (lambda (s)
- (if s
- (string-trim s)
- ""))
- (nreverse result)))))
+ (when-let* ((func (or (plist-get args :function)
elisa-semantic-split-function))
+ (k (or (plist-get args :threshold-amount)
elisa-breakpoint-threshold-amount))
+ (chunks (funcall func))
+ (embeddings (cl-remove-if
+ #'not
+ (mapcar (lambda (s)
+ (when (length> (string-trim s) 0)
+ (llm-embedding elisa-embeddings-provider
s)))
+ chunks)))
+ (distances (elisa--distances embeddings))
+ (threshold (elisa-calculate-threshold k distances))
+ (current (car chunks))
+ (tail (cdr chunks)))
+ (let* ((result nil))
+ (mapc
+ (lambda (el)
+ (if (<= el threshold)
+ (setq current (concat current (car tail)))
+ (push current result)
+ (setq current (car tail)))
+ (setq tail (cdr tail)))
+ distances)
+ (push current result)
+ (cl-remove-if
+ #'string-empty-p
+ (mapcar (lambda (s)
+ (if s
+ (string-trim s)
+ ""))
+ (nreverse result))))))
(defun elisa-search-duckduckgo (prompt)
"Search duckduckgo for PROMPT and return list of urls."
@@ -531,7 +531,10 @@ You can customize `elisa-searxng-url' to use non local
instance."
(plz 'get url :as 'buffer
:headers `(("Accept" . ,eww-accept-content-types)
("Accept-Encoding" . "gzip")
- ("User-Agent" .
,(url-http--user-agent-default-string)))))))
+ ("User-Agent" .
,(url-http--user-agent-default-string))))))
+ ;; fix one word lines for async execution
+ (shr-use-fonts nil)
+ (shr-width (- ellama-long-lines-length 5)))
(when buffer-name
(with-current-buffer buffer-name
(goto-char (point-min))
@@ -814,6 +817,7 @@ WHERE d.rowid in %s;"
,(async-inject-variables "elisa-web-pages-limit")
,(async-inject-variables "elisa-breakpoint-threshold-amount")
,(async-inject-variables "elisa-pandoc-executable")
+ ,(async-inject-variables "ellama-long-lines-length")
,(async-inject-variables "load-path")
(require 'elisa)
(,func))
- [elpa] externals/elisa 27fc248901 71/98: Remove files metadata on collection remove, (continued)
- [elpa] externals/elisa 27fc248901 71/98: Remove files metadata on collection remove, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa 5bf5d63fc1 07/98: Improve defaults, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa 61dd36890a 21/98: Enable load file check in CI, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa 1f5393d37e 22/98: Warn user instead of network call during package loading, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa 80401a0b52 30/98: Search for gzipped builtin manuals, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa c59d491e18 35/98: Add prompt rewriting with ellama-chain, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa 3874a7007c 36/98: Improve semantic split api, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa f2bf34b201 39/98: Improve sqlite escape function, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa 45b854ba2d 46/98: Fix linter warning, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa d58e172912 48/98: Make web search async, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa f312d189f2 51/98: Fix one word lines in webpage quotes parsed asyncronously,
ELPA Syncer <=
- [elpa] externals/elisa 0fc73b4b9a 69/98: Add instruction to elisa-chat, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa 4710f87851 72/98: Refactor parsing info manuals, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa 86f4ea0afb 74/98: Fix parsing info manuals with infinite loop, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa 9c7004e15c 77/98: Add functionality to parse web pages and add them to collections, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa 4063c45908 78/98: Add custom variables documentation, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa 0c432dd1c8 81/98: Add commands to documentation, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa 29c17f86a7 83/98: Update package description, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa cb69d24b0d 88/98: Improve documentation, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa 8d0ad42134 90/98: Small documentation markup fixes, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa 55200f8d8b 89/98: Fix readme markup for github rendering, ELPA Syncer, 2024/07/17