[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[elpa] externals/elisa b419fb2cf2 61/98: Add code for parsing directory
From: |
ELPA Syncer |
Subject: |
[elpa] externals/elisa b419fb2cf2 61/98: Add code for parsing directory as an elisa collection |
Date: |
Wed, 17 Jul 2024 18:58:04 -0400 (EDT) |
branch: externals/elisa
commit b419fb2cf2bd17e39d233f36c66b968af11a69e2
Author: Sergey Kostyaev <kostyaev.sergey2@wb.ru>
Commit: Sergey Kostyaev <kostyaev.sergey2@wb.ru>
Add code for parsing directory as an elisa collection
---
elisa.el | 86 +++++++++++++++++++++++++++++++++++++++++-----------------------
1 file changed, 55 insertions(+), 31 deletions(-)
diff --git a/elisa.el b/elisa.el
index b106a55c56..3b713923ea 100644
--- a/elisa.el
+++ b/elisa.el
@@ -668,12 +668,12 @@ than T, it will be packed into single semantic chunk."
"Parse file PATH for COLLECTION-ID.
When FORCE parse even if already parsed."
(let* ((opened (get-file-buffer path))
- (buf (find-file-noselect path t t))
+ (buf (or opened (find-file-noselect path t t)))
(hash (secure-hash 'sha256 buf))
- (prev-hash (flatten-tree (sqlite-select
- elisa-db
- (format "select hash from files where path =
'%s';"
- (elisa-sqlite-escape path))))))
+ (prev-hash (caar (sqlite-select
+ elisa-db
+ (format "select hash from files where path = '%s';"
+ (elisa-sqlite-escape path))))))
(when (or force
(not prev-hash)
(not (string-equal hash prev-hash)))
@@ -747,6 +747,26 @@ When FORCE parse even if already parsed."
(when (not opened)
(kill-buffer buf))))
+(defun elisa-parse-directory (dir)
+ "Parse DIR as new collection syncronously."
+ (interactive "DSelect directory: ")
+ (setq dir (expand-file-name dir))
+ (let ((collection-id (progn
+ (sqlite-execute
+ elisa-db
+ (format
+ "insert into collections (name) values ('%s') on
conflict do nothing;"
+ (elisa-sqlite-escape dir)))
+ (caar (sqlite-select
+ elisa-db
+ (format
+ "select rowid from collections where name =
'%s';"
+ (elisa-sqlite-escape dir)))))))
+ (mapc (lambda (file)
+ (message "parsing %s" file)
+ (elisa-parse-file collection-id file))
+ (elisa--file-list dir))))
+
(defun elisa-search-duckduckgo (prompt)
"Search duckduckgo for PROMPT and return list of urls."
(let* ((url (format "https://duckduckgo.com/html/?q=%s"; (url-hexify-string
prompt)))
@@ -849,14 +869,15 @@ You can customize `elisa-searxng-url' to use non local
instance."
(defun elisa--do-rerank-request (prompt ids)
"Call rerank service for PROMPT and IDS."
- (seq--into-list
- (alist-get 'data
- (plz 'post (format "%s/api/v1/rerank"
- (string-remove-suffix "/" elisa-reranker-url))
- :headers `(("Content-Type" . "application/json"))
- :body-type 'text
- :body (elisa--rerank-request prompt ids)
- :as #'json-read))))
+ (when ids
+ (seq--into-list
+ (alist-get 'data
+ (plz 'post (format "%s/api/v1/rerank"
+ (string-remove-suffix "/"
elisa-reranker-url))
+ :headers `(("Content-Type" . "application/json"))
+ :body-type 'text
+ :body (elisa--rerank-request prompt ids)
+ :as #'json-read)))))
(defun elisa-rerank (prompt ids)
"Rerank IDS according to PROMPT and return top `elisa-limit' IDS."
@@ -946,28 +967,30 @@ Return sqlite query that extract data for adding to
context."
(ids (if elisa-reranker-enabled
(elisa-rerank prompt raw-ids)
(take elisa-limit raw-ids))))
- (sqlite-select
- elisa-db
- (format
- "SELECT k.name, d.path, d.data
+ (when ids
+ (sqlite-select
+ elisa-db
+ (format
+ "SELECT k.name, d.path, d.data
FROM data AS d
LEFT JOIN kinds k ON k.rowid = d.kind_id
WHERE d.rowid in %s;"
- (elisa-sqlite-format-int-list ids)))))
+ (elisa-sqlite-format-int-list ids))))))
(lambda (result)
- (mapc
- (lambda (row)
- (when-let ((kind (cl-first row))
- (path (cl-second row))
- (text (cl-third row)))
- (pcase kind
- ("web"
- (ellama-context-add-webpage-quote-noninteractive path path text))
- ("file"
- (ellama-context-add-file-quote-noninteractive path text))
- ("info"
- (ellama-context-add-info-node-quote-noninteractive path text)))))
- result)
+ (if result (mapc
+ (lambda (row)
+ (when-let ((kind (cl-first row))
+ (path (cl-second row))
+ (text (cl-third row)))
+ (pcase kind
+ ("web"
+ (ellama-context-add-webpage-quote-noninteractive path
path text))
+ ("file"
+ (ellama-context-add-file-quote-noninteractive path
text))
+ ("info"
+ (ellama-context-add-info-node-quote-noninteractive path
text)))))
+ result)
+ (ellama-context-add-text "No related documents found."))
(ellama-chat prompt nil :provider elisa-chat-provider))))
(defun elisa-get-builtin-manuals ()
@@ -1039,6 +1062,7 @@ Call ON-DONE callback with result as an argument after
FUNC evaluation done."
,(async-inject-variables "elisa-breakpoint-threshold-amount")
,(async-inject-variables "elisa-pandoc-executable")
,(async-inject-variables "ellama-long-lines-length")
+ ,(async-inject-variables "elisa-reranker-enabled")
,(async-inject-variables "load-path")
(require 'elisa)
(,func))
- [elpa] externals/elisa d90df5889d 38/98: Add function that return buffer with url content, (continued)
- [elpa] externals/elisa d90df5889d 38/98: Add function that return buffer with url content, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa 0f94c23a5d 40/98: Add more sqlite tables, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa e877f8f5c7 44/98: First implementation for web search, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa 5ca66e9f0d 47/98: Fix custom variables, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa 273a1d492d 50/98: Add reranker to RAG pipeline, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa ade7ac0af9 52/98: Update info manual parsing, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa 8a2c92dc34 54/98: Fix parsing info manuals, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa cecc5cb13f 55/98: Make sync parsing interactive, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa 4cad3085fd 57/98: Use more async calls to prevent emacs from blocking, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa ad130b564f 60/98: Add parse file function, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa b419fb2cf2 61/98: Add code for parsing directory as an elisa collection,
ELPA Syncer <=
- [elpa] externals/elisa ef06534f46 62/98: Implement incremental parsing, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa 0e32d7bb5c 63/98: Add async directory parsing, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa f744ce305a 67/98: Add reparse current collection command, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa 9ad7827337 70/98: Fix semantic split with single chunk, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa 439ed1d4f8 76/98: Make executable customization simpler, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa e5691f59c5 80/98: Make syncronous functions non-interactive, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa e92628390b 82/98: Update example configuration, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa 39915439a4 84/98: Update installation instructions, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa fbfe3b4ae1 86/98: Merge pull request #12 from s-kostyaev/semantic-split, ELPA Syncer, 2024/07/17
- [elpa] externals/elisa 3882b9b322 87/98: Bump version, ELPA Syncer, 2024/07/17