[Emacs-diffs] Changes to emacs/lisp/mh-e/mh-junk.el [lexbind]

[Miles Bader]

Index: emacs/lisp/mh-e/mh-junk.el
diff -c /dev/null emacs/lisp/mh-e/mh-junk.el:1.2.2.1
*** /dev/null   Tue Oct 14 19:39:48 2003
--- emacs/lisp/mh-e/mh-junk.el  Tue Oct 14 19:39:26 2003
***************
*** 0 ****
--- 1,417 ----
+ ;;; mh-junk.el --- Interface to anti-spam measures
+ 
+ ;; Copyright (C) 2003 Free Software Foundation, Inc.
+ 
+ ;; Author: Satyaki Das <address@hidden>,
+ ;;         Bill Wohler <address@hidden>
+ ;; Maintainer: Bill Wohler <address@hidden>
+ ;; Keywords:  mail, spam
+ 
+ ;; This file is part of GNU Emacs.
+ 
+ ;; GNU Emacs is free software; you can redistribute it and/or modify
+ ;; it under the terms of the GNU General Public License as published by
+ ;; the Free Software Foundation; either version 2, or (at your option)
+ ;; any later version.
+ 
+ ;; GNU Emacs is distributed in the hope that it will be useful,
+ ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+ ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ ;; GNU General Public License for more details.
+ 
+ ;; You should have received a copy of the GNU General Public License
+ ;; along with GNU Emacs; see the file COPYING.  If not, write to the
+ ;; Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ ;; Boston, MA 02111-1307, USA.
+ 
+ ;;; Commentary:
+ 
+ ;; Spam handling in MH-E.
+ 
+ ;;; Change Log:
+ 
+ ;;; Code:
+ 
+ (require 'mh-e)
+ 
+ ;; Interactive functions callable from the folder buffer
+ ;;;###mh-autoload
+ (defun mh-junk-blacklist (msg-or-seq)
+   "Blacklist MSG-OR-SEQ as spam.
+ Default is the displayed message.
+ If optional prefix argument is provided, then prompt for the message sequence.
+ If variable `transient-mark-mode' is non-nil and the mark is active, then the
+ selected region is blacklisted.
+ In a program, MSG-OR-SEQ can be a message number, a list of message numbers, a
+ region in a cons cell, or a sequence.
+ 
+ First the appropriate function is called depending on the value of
+ `mh-junk-choice'. Then if `mh-junk-mail-folder' is a string then the message 
is
+ refiled to that folder. If nil, the message is deleted.
+ 
+ To change the spam program being used, customize `mh-junk-program'. Directly
+ setting `mh-junk-choice' is not recommended.
+ 
+ The documentation for the following functions describes what setup is needed
+ for the different spam fighting programs:
+ 
+   - `mh-bogofilter-blacklist'
+   - `mh-spamprobe-blacklist'
+   - `mh-spamassassin-blacklist'"
+   (interactive (list (mh-interactive-msg-or-seq "Blacklist")))
+   (let ((blacklist-func (nth 1 (assoc mh-junk-choice 
mh-junk-function-alist))))
+     (unless blacklist-func
+       (error "Customize `mh-junk-program' appropriately"))
+     (let ((dest (cond ((null mh-junk-mail-folder) nil)
+                       ((equal mh-junk-mail-folder "") "+")
+                       ((eq (aref mh-junk-mail-folder 0) ?+)
+                        mh-junk-mail-folder)
+                       ((eq (aref mh-junk-mail-folder 0) ?@)
+                        (concat mh-current-folder "/"
+                                (substring mh-junk-mail-folder 1)))
+                       (t (concat "+" mh-junk-mail-folder)))))
+       (mh-iterate-on-msg-or-seq msg msg-or-seq
+         (funcall (symbol-function blacklist-func) msg)
+         (if dest
+             (mh-refile-a-msg nil (intern dest))
+           (mh-delete-a-msg nil)))
+       (mh-next-msg))))
+ 
+ ;;;###mh-autoload
+ (defun mh-junk-whitelist (msg-or-seq)
+   "Whitelist MSG-OR-SEQ incorrectly classified as spam.
+ Default is the displayed message.
+ If optional prefix argument is provided, then prompt for the message sequence.
+ If variable `transient-mark-mode' is non-nil and the mark is active, then the
+ selected region is whitelisted.
+ In a program, MSG-OR-SEQ can be a message number, a list of message numbers, a
+ region in a cons cell, or a sequence.
+ 
+ First the appropriate function is called depending on the value of
+ `mh-junk-choice'. Then the message is refiled to `mh-inbox'.
+ 
+ To change the spam program being used, customize `mh-junk-program'. Directly
+ setting `mh-junk-choice' is not recommended."
+   (interactive (list (mh-interactive-msg-or-seq "Whitelist")))
+   (let ((whitelist-func (nth 2 (assoc mh-junk-choice 
mh-junk-function-alist))))
+     (unless whitelist-func
+       (error "Customize `mh-junk-program' appropriately"))
+     (mh-iterate-on-msg-or-seq msg msg-or-seq
+       (funcall (symbol-function whitelist-func) msg)
+       (mh-refile-a-msg nil (intern mh-inbox)))
+     (mh-next-msg)))
+ 
+ 
+ 
+ ;; Bogofilter Interface
+ 
+ (defvar mh-bogofilter-executable (executable-find "bogofilter"))
+ 
+ (defun mh-bogofilter-blacklist (msg)
+   "Classify MSG as spam.
+ Tell bogofilter that the message is spam.
+ 
+ Bogofilter is a Bayesian spam filtering program. Get it from your local
+ distribution or from:
+    http://bogofilter.sourceforge.net/
+ 
+ You first need to teach bogofilter. This is done by running
+ 
+    bogofilter -n < good-message
+ 
+ on every good message, and
+ 
+    bogofilter -s < spam-message
+ 
+ on every spam message. Most Bayesian filters need 1000 to 5000 of each to
+ start doing a good job.
+ 
+ To use bogofilter, add the following .procmailrc recipes which you can also
+ find in the bogofilter man page:
+ 
+    # Bogofilter
+    :0fw
+    | bogofilter -u -e -p
+ 
+    :0
+    * ^X-Bogosity: Yes, tests=bogofilter
+    $SPAM
+ 
+ Bogofilter continues to feed the messages it classifies back into its
+ database. Occasionally it misses, and those messages need to be reclassified.
+ MH-E can do this for you. Use \\[mh-junk-blacklist] to reclassify messges in
+ your +inbox as spam, and \\[mh-junk-whitelist] to reclassify messages in your
+ spambox as good messages."
+   (unless mh-bogofilter-executable
+     (error "Couldn't find the bogofilter executable"))
+   (let ((msg-file (mh-msg-filename msg mh-current-folder)))
+     (call-process mh-bogofilter-executable msg-file 0 nil "-Ns")))
+ 
+ (defun mh-bogofilter-whitelist (msg)
+   "Reinstate incorrectly filtered MSG.
+ Train bogofilter to think of the message as non-spam."
+   (unless mh-bogofilter-executable
+     (error "Couldn't find the bogofilter executable"))
+   (let ((msg-file (mh-msg-filename msg mh-current-folder)))
+     (call-process mh-bogofilter-executable msg-file 0 nil "-Sn")))
+ 
+ 
+ 
+ ;; Spamprobe Interface
+ 
+ (defvar mh-spamprobe-executable (executable-find "spamprobe"))
+ 
+ (defun mh-spamprobe-blacklist (msg)
+   "Classify MSG as spam.
+ Tell spamprobe that the message is spam.
+ 
+ Spamprobe is a Bayesian spam filtering program. More info about the program 
can
+ be found at:
+    http://spamprobe.sourceforge.net
+ 
+ Here is a procmail recipe to stores incoming spam mail into the folder +spam
+ and good mail in /home/user/Mail/mdrop/mbox. This recipe is provided as an
+ example in the spamprobe man page.
+ 
+    PATH=/bin:/usr/bin:/usr/local/bin
+    DEFAULT=/home/user/Mail/mdrop/mbox
+    SPAM=/home/user/Mail/spam/.
+ 
+    # Spamprobe filtering
+    :0
+    SCORE=| spamprobe receive
+    :0 wf
+    | formail -I \"X-SpamProbe: $SCORE\"
+    :0 a:
+    *^X-SpamProbe: SPAM
+    $SPAM
+ 
+ Occasionally some good mail gets misclassified as spam. You can use
+ \\[mh-junk-whitelist] to reclassify that as good mail."
+   (unless mh-spamprobe-executable
+     (error "Couldn't find the spamprobe executable"))
+   (let ((msg-file (mh-msg-filename msg mh-current-folder)))
+     (call-process mh-spamprobe-executable msg-file 0 nil "spam")))
+ 
+ (defun mh-spamprobe-whitelist (msg)
+   "Reinstate incorrectly filtered MSG.
+ Train spamprobe to think of the message as non-spam."
+   (unless mh-spamprobe-executable
+     (error "Couldn't find the spamprobe executable"))
+   (let ((msg-file (mh-msg-filename msg mh-current-folder)))
+     (call-process mh-spamprobe-executable msg-file 0 nil "good")))
+ 
+ 
+ 
+ ;; Spamassassin Interface
+ 
+ (defvar mh-spamassassin-executable (executable-find "spamassassin"))
+ (defvar mh-sa-learn-executable (executable-find "sa-learn"))
+ 
+ (defun mh-spamassassin-blacklist (msg)
+   "Blacklist MSG.
+ This is done by sending the message to Razor and by appending the sender to
+ ~/.spamassassin/user_prefs in a blacklist_from rule. If sa-learn is available,
+ the message is also recategorized as spam.
+ 
+ Spamassassin is an excellent spam filter. For more information, see:
+   http://spamassassin.org/.
+ 
+ I ran \"spamassassin -t\" on every mail message in my archive and ran an
+ analysis in Gnumeric to find that the standard deviation of good mail
+ scored under 5 (coincidentally, the spamassassin default for \"spam\").
+ 
+ Furthermore, I observed that there weren't any messages with a score of 8
+ or more that were interesting, so I added a couple of points to be
+ conservative and send any message with a score of 10 or more down the
+ drain. You might want to use a score of 12 or 13 to be really conservative.
+ I have found that this really decreases the amount of junk to review.
+ 
+ Messages with a score of 5-9 are set aside for later review. The major
+ weakness of rules-based filters is a plethora of false positives\; I catch one
+ or two legitimate messages in here a week, so it is worthwhile to check.
+ 
+ You might choose to do this analysis yourself to pick a good score for
+ deleting spam sight unseen, or you might pick a score out of a hat, or you
+ might choose to be very conservative and not delete any messages at all.
+ 
+ Based upon this discussion, here is what the associated ~/.procmailrc
+ entries look like. These rules appear before my list filters so that spam
+ sent to mailing lists gets pruned too.
+ 
+    #
+    # Spam
+    #
+    :0fw
+    | spamc
+ 
+    # Anything with a spam level of 10 or more is junked immediately.
+    :0:
+    * ^X-Spam-Level: ..........
+    /dev/null
+ 
+    :0
+    * ^X-Spam-Status: Yes
+    $SPAM
+ 
+ If you don't use \"spamc\", use \"spamassassin -P -a\".
+ 
+ A handful of spam does find its way into +inbox. In this case, use
+ \\[mh-junk-blacklist] to add a \"blacklist_from\" line to
+ ~/spamassassin/user_prefs, delete the message, and send the message to the
+ Razor, so that others might not see this spam.
+ 
+ Over time, you see some patterns in the blacklisted addresses and can
+ replace several lines with wildcards. For example, it is clear that High
+ Speed Media is the biggest bunch of jerks on the Net. Here are some of the
+ entries I have for them, and the list continues to grow.
+ 
+    blacklist_from     address@hidden
+    blacklist_from     address@hidden
+    blacklist_from     address@hidden
+    blacklist_from     address@hidden
+    blacklist_from     address@hidden
+    blacklist_from     address@hidden
+    blacklist_from     address@hidden
+    blacklist_from     address@hidden
+    blacklist_from     address@hidden
+    blacklist_from     address@hidden
+    blacklist_from     address@hidden
+ 
+ The function `mh-spamassassin-identify-spammers' is provided that shows the
+ frequency counts of the host and domain names in your blacklist_from
+ entries. This can be helpful when editing the blacklist_from entries.
+ 
+ In versions of spamassassin (2.50 and on) that support a Bayesian classifier,
+ \\[mh-junk-blacklist] uses the sa-learn program to recategorize the message as
+ spam. Neither MH-E, nor spamassassin, rebuilds the database after adding
+ words, so you will need to run \"sa-learn --rebuild\" periodically. This can
+ be done by adding the following to your crontab:
+ 
+   0 * * * *   sa-learn --rebuild > /dev/null 2>&1"
+   (unless mh-spamassassin-executable
+     (error "Couldn't find the spamassassin executable"))
+   (let ((current-folder mh-current-folder)
+         (msg-file (mh-msg-filename msg mh-current-folder))
+         (sender))
+     (save-excursion
+       (message "Giving this message the Razor...")
+       (mh-truncate-log-buffer)
+       (call-process mh-spamassassin-executable msg-file mh-log-buffer nil
+                     "--report" "--remove-from-whitelist")
+       (when mh-sa-learn-executable
+           (message "Recategorizing this message as spam...")
+           (call-process mh-sa-learn-executable msg-file mh-log-buffer nil
+                         "--single" "--spam" "--local --no-rebuild"))
+       (message "Blacklisting address...")
+       (set-buffer (get-buffer-create mh-temp-buffer))
+       (erase-buffer)
+       (call-process (expand-file-name mh-scan-prog mh-progs) nil t nil
+                     (format "%s" msg) current-folder
+                     "-format" "%<(mymbox{from})%|%(addr{from})%>")
+       (goto-char (point-min))
+       (if (search-forward-regexp "^\\(.+\\)$" nil t)
+           (progn
+             (setq sender (match-string 0))
+             (mh-spamassassin-add-rule "blacklist_from" sender)
+             (message "Blacklisting address...done"))
+         (message "Blacklisting address...not done (from my address)")))))
+ 
+ (defun mh-spamassassin-whitelist (msg)
+   "Whitelist MSG.
+ Add a whitelist_from rule to the ~/.spamassassin/user_prefs file. If sa-learn
+ is available, then the message is recategorized as ham."
+   (unless mh-spamassassin-executable
+     (error "Couldn't find the spamassassin executable"))
+   (let ((msg-file (mh-msg-filename msg mh-current-folder))
+         (show-buffer (get-buffer mh-show-buffer))
+         from)
+     (save-excursion
+       (set-buffer (get-buffer-create mh-temp-buffer))
+       (erase-buffer)
+       (message "Removing spamassassin markup from message...")
+       (call-process mh-spamassassin-executable msg-file mh-temp-buffer nil
+                     "--remove-markup")
+       (if show-buffer
+           (kill-buffer show-buffer))
+       (write-file msg-file)
+       (when mh-sa-learn-executable
+         (message "Recategorizing this message as ham...")
+         (call-process mh-sa-learn-executable msg-file mh-temp-buffer nil
+                       "--single" "--ham" "--local --no-rebuild"))
+       (message "Whitelisting address...")
+       (setq from (car (ietf-drums-parse-address (mh-get-header-field 
"From:"))))
+       (kill-buffer nil)
+       (unless (equal from "")
+         (mh-spamassassin-add-rule "whitelist_from" from))
+       (message "Whitelisting address...done"))))
+ 
+ (defun mh-spamassassin-add-rule (rule body)
+   "Add a new rule to ~/.spamassassin/user_prefs.
+ The name of the rule is RULE and its body is BODY."
+   (save-window-excursion
+     (let* ((line (format "%s\t%s\n" rule body))
+            (case-fold-search t)
+            (file (expand-file-name "~/.spamassassin/user_prefs"))
+            (buffer-exists (find-buffer-visiting file)))
+       (find-file file)
+       (if (not (search-forward (format "\n%s" line) nil t))
+           (progn
+             (goto-char (point-max))
+             (insert (if (bolp) "" "\n") line)
+             (save-buffer)))
+       (if (not buffer-exists)
+           (kill-buffer nil)))))
+ 
+ (defun mh-spamassassin-identify-spammers ()
+   "Identifies spammers who are repeat offenders.
+ 
+ For each blacklist_from entry from the last blank line of
+ ~/.spamassassin/user_prefs to the end of the file, a list of host and domain
+ names along with their frequency counts is displayed. This information can be
+ used to replace multiple blacklist_from entries with a single wildcard entry
+ such as:
+ 
+   blacklist_from address@hidden"
+   (interactive)
+   (let* ((file (expand-file-name "~/.spamassassin/user_prefs"))
+          (domains (make-hash-table :test 'equal)))
+     (find-file file)
+     ;; Only consider entries between last blank line and end of file.
+     (goto-char (1- (point-max)))
+     (search-backward-regexp "^$")
+     ;; Perform frequency count.
+     (save-excursion
+       (while (search-forward-regexp "^blacklist_from\\s-*\\(.*\\)@\\(.*\\)$"
+                                     nil t)
+         (let ((host (match-string 2))
+               value)
+           ;; Remove top-level-domain from hostname.
+           (setq host (cdr (reverse (split-string host "\\."))))
+           ;; Add counts for each host and domain part.
+           (while host
+             (setq value (gethash (car host) domains))
+             (puthash (car host) (1+ (if (not value) 0 value)) domains)
+             (setq host (cdr host))))))
+ 
+     ;; Output
+     (delete-other-windows)
+     (pop-to-buffer (get-buffer-create "*MH-E Spammer Frequencies*"))
+     (erase-buffer)
+     (maphash '(lambda (key value) ""
+                 (if (> value 2)
+                     (insert (format "%s %s\n" key value))))
+              domains)
+     (sort-numeric-fields 2 (point-min) (point-max))
+     (reverse-region (point-min) (point-max))
+     (goto-char (point-min))))
+ 
+ (provide 'mh-junk)
+ 
+ ;;; Local Variables:
+ ;;; indent-tabs-mode: nil
+ ;;; sentence-end-double-space: nil
+ ;;; End:
+ 
+ ;;; arch-tag: 603335f1-77ff-4306-8828-5d3dad51abe1
+ ;;; mh-junk.el ends here