]> Pileus Git - ~andy/fetchmail/blobdiff - driver.c
Note Earl's regression fix for SSL_CTX_clear_options() on older OpenSSL.
[~andy/fetchmail] / driver.c
index 4a4f4d3faee0d118866c2ca8063b307048acaafc..c2917268cedfce70fa9d70f085bad27b7f4f201b 100644 (file)
--- a/driver.c
+++ b/driver.c
 #if defined(HAVE_SYS_ITIMER_H)
 #include <sys/itimer.h>
 #endif
-#include  <sys/time.h>
 #include  <signal.h>
 #ifdef HAVE_SYS_WAIT_H
 #include <sys/wait.h>
 #endif
 
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
 #ifdef HAVE_NET_SOCKET_H
 #include <net/socket.h>
 #endif
-#ifdef HESIOD
+#include <netdb.h>
+#ifdef HAVE_PKG_hesiod
+#ifdef __cplusplus
+extern "C" {
+#endif
 #include <hesiod.h>
+#ifdef __cplusplus
+}
+#endif
 #endif
 
-#if defined(HAVE_RES_SEARCH) || defined(HAVE_GETHOSTBYNAME)
-#include <netdb.h>
-#include "mx.h"
-#endif /* defined(HAVE_RES_SEARCH) || defined(HAVE_GETHOSTBYNAME) */
+#include <langinfo.h>
 
 #include "kerberos.h"
 #ifdef KERBEROS_V4
 #include "socket.h"
 
 #include "fetchmail.h"
+#include "getaddrinfo.h"
 #include "tunable.h"
 
+#include "sdump.h"
+
 /* throw types for runtime errors */
 #define THROW_TIMEOUT  1               /* server timed out */
-#define THROW_SIGPIPE  2               /* SIGPIPE on stream socket */
 
 /* magic values for the message length array */
 #define MSGLEN_UNKNOWN 0               /* length unknown (0 is impossible) */
@@ -69,14 +77,28 @@ int batchcount;             /* count of messages sent in current batch */
 flag peek_capable;     /* can we peek for better error recovery? */
 int mailserver_socket_temp = -1;       /* socket to free if connect timeout */ 
 
-static int timeoutcount;               /* count consecutive timeouts */
+struct addrinfo *ai0, *ai1;    /* clean these up after signal */
+
+static volatile int timeoutcount = 0;  /* count consecutive timeouts */
+static volatile int idletimeout = 0;   /* timeout occured in idle stage? */
+
+static sigjmp_buf      restart;
 
-static jmp_buf restart;
+int is_idletimeout(void)
+/* last timeout occured in idle stage? */
+{
+    return idletimeout;
+}
+
+void resetidletimeout(void)
+{
+    idletimeout = 0;
+}
 
 void set_timeout(int timeleft)
 /* reset the nonresponse-timeout */
 {
-#if !defined(__EMX__) && !defined(__BEOS__) 
+#if !defined(__EMX__) && !defined(__BEOS__)
     struct itimerval ntimeout;
 
     if (timeleft == 0)
@@ -92,14 +114,15 @@ void set_timeout(int timeleft)
 static RETSIGTYPE timeout_handler (int signal)
 /* handle SIGALRM signal indicating a server timeout */
 {
-    timeoutcount++;
-    longjmp(restart, THROW_TIMEOUT);
-}
-
-static RETSIGTYPE sigpipe_handler (int signal)
-/* handle SIGPIPE signal indicating a broken stream socket */
-{
-    longjmp(restart, THROW_SIGPIPE);
+    (void)signal;
+    if(stage != STAGE_IDLE) {
+       timeoutcount++;
+       /* XXX FIXME: this siglongjmp must die - it's not safe to be
+        * called from a function handler and breaks, for instance,
+        * getaddrinfo() */
+       siglongjmp(restart, THROW_TIMEOUT);
+    } else
+       idletimeout = 1;
 }
 
 #define CLEANUP_TIMEOUT 60 /* maximum timeout during cleanup */
@@ -124,7 +147,6 @@ int socket;         /* socket to server host */
 char *canonical;       /* server name */
 char *principal;
 {
-    char * host_primary;
     KTEXT ticket;
     MSG_DAT msg_data;
     CREDENTIALS cred;
@@ -155,7 +177,7 @@ char *principal;
        }
     }
   
-    xalloca(ticket, KTEXT, sizeof (KTEXT_ST));
+    ticket = xmalloc(sizeof (KTEXT_ST));
     rem = (krb_sendauth (0L, socket, ticket,
                         prin ? prin : "pop",
                         inst ? inst : canonical,
@@ -167,6 +189,7 @@ char *principal;
                         ((struct sockaddr_in *) 0),
                         ((struct sockaddr_in *) 0),
                         "KPOPV0.1"));
+    free(ticket);
     if (prin_copy)
     {
         free(prin_copy);
@@ -195,22 +218,21 @@ const char *canonical;  /* server name */
     krb5_auth_context auth_context = NULL;
 
     krb5_init_context(&context);
-    krb5_init_ets(context);
     krb5_auth_con_init(context, &auth_context);
 
-    if (retval = krb5_cc_default(context, &ccdef)) {
+    if ((retval = krb5_cc_default(context, &ccdef))) {
         report(stderr, "krb5_cc_default: %s\n", error_message(retval));
         return(PS_ERROR);
     }
 
-    if (retval = krb5_cc_get_principal(context, ccdef, &client)) {
+    if ((retval = krb5_cc_get_principal(context, ccdef, &client))) {
         report(stderr, "krb5_cc_get_principal: %s\n", error_message(retval));
         return(PS_ERROR);
     }
 
-    if (retval = krb5_sname_to_principal(context, canonical, "pop",
+    if ((retval = krb5_sname_to_principal(context, canonical, "pop",
            KRB5_NT_UNKNOWN,
-           &server)) {
+           &server))) {
         report(stderr, "krb5_sname_to_principal: %s\n", error_message(retval));
         return(PS_ERROR);
     }
@@ -232,15 +254,17 @@ const char *canonical;  /* server name */
     if (retval) {
 #ifdef HEIMDAL
       if (err_ret && err_ret->e_text) {
-          report(stderr, GT_("krb5_sendauth: %s [server says '%*s'] \n"),
-                 error_message(retval),
-                 err_ret->e_text);
+         char *t = err_ret->e_text;
+         char *tt = sdump(t, strlen(t));
+          report(stderr, GT_("krb5_sendauth: %s [server says '%s']\n"),
+                 error_message(retval), tt);
+         free(tt);
 #else
       if (err_ret && err_ret->text.length) {
-          report(stderr, GT_("krb5_sendauth: %s [server says '%*s'] \n"),
-                error_message(retval),
-                err_ret->text.length,
-                err_ret->text.data);
+         char *tt = sdump(err_ret->text.data, err_ret->text.length);
+          report(stderr, GT_("krb5_sendauth: %s [server says '%s']\n"),
+                error_message(retval), tt);
+         free(tt);
 #endif
          krb5_free_error(context, err_ret);
       } else
@@ -315,14 +339,22 @@ static void send_size_warnings(struct query *ctl)
      * but it's not a disaster, either, since the skipped mail will not
      * be deleted.
      */
-    if (open_warning_by_mail(ctl, (struct msgblk *)NULL))
+    if (open_warning_by_mail(ctl))
        return;
-    stuff_warning(ctl,
-          GT_("Subject: Fetchmail oversized-messages warning.\n"
-            "\n"
-            "The following oversized messages remain on the mail server %s:"),
-                 ctl->server.pollname);
+    stuff_warning(iana_charset, ctl,
+          GT_("Subject: Fetchmail oversized-messages warning"));
+    stuff_warning(NULL, ctl, "%s", "");
+    if (ctl->limitflush)
+       stuff_warning(NULL, ctl,
+               GT_("The following oversized messages were deleted on server %s account %s:"),
+               ctl->server.pollname, ctl->remotename);
+    else
+       stuff_warning(NULL, ctl,
+               GT_("The following oversized messages remain on server %s account %s:"),
+               ctl->server.pollname, ctl->remotename);
+
+    stuff_warning(NULL, ctl, "%s", "");
+
     if (run.poll_interval == 0)
        max_warning_poll_count = 0;
     else
@@ -335,9 +367,16 @@ static void send_size_warnings(struct query *ctl)
        {
            nbr = current->val.status.mark;
            size = atoi(current->id);
-           stuff_warning(ctl, 
-                   GT_("\t%d msg %d octets long skipped by fetchmail.\n"),
-                   nbr, size);
+           if (ctl->limitflush)
+               stuff_warning(NULL, ctl,
+                       ngettext("  %d message  %d octets long deleted by fetchmail.",
+                                "  %d messages %d octets long deleted by fetchmail.", nbr),
+                       nbr, size);
+           else
+               stuff_warning(NULL, ctl,
+                       ngettext("  %d message  %d octets long skipped by fetchmail.",
+                                "  %d messages %d octets long skipped by fetchmail.", nbr),
+                       nbr, size);
        }
        current->val.status.num++;
        current->val.status.mark = 0;
@@ -346,10 +385,12 @@ static void send_size_warnings(struct query *ctl)
            current->val.status.num = 0;
     }
 
+    stuff_warning(NULL, ctl, "%s", "");
+
     close_warning_by_mail(ctl, (struct msgblk *)NULL);
 }
 
-static void mark_oversized(struct query *ctl, int num, int size)
+static void mark_oversized(struct query *ctl, int size)
 /* mark a message oversized */
 {
     struct idlist *current=NULL, *tmp=NULL;
@@ -357,12 +398,7 @@ static void mark_oversized(struct query *ctl, int num, int size)
     int cnt;
 
     /* convert size to string */
-#ifdef HAVE_SNPRINTF
-    snprintf(sizestr, sizeof(sizestr),
-#else
-    sprintf(sizestr,
-#endif /* HAVE_SNPRINTF */
-      "%d", size);
+    snprintf(sizestr, sizeof(sizestr), "%d", size);
 
     /* build a list of skipped messages
      * val.id = size of msg (string cnvt)
@@ -379,16 +415,9 @@ static void mark_oversized(struct query *ctl, int num, int size)
     cnt = current ? current->val.status.num : 0;
 
     /* if entry exists, increment the count */
-    if (current && str_in_list(&current, sizestr, FALSE))
+    if (current && (tmp = str_in_list(&current, sizestr, FALSE)))
     {
-       for ( ; current; current = current->next)
-       {
-           if (strcmp(current->id, sizestr) == 0)
-           {
-               current->val.status.mark++;
-               break;
-           }
-       }
+       tmp->val.status.mark++;
     }
     /* otherwise, create a new entry */
     /* initialise with current poll count */
@@ -399,12 +428,70 @@ static void mark_oversized(struct query *ctl, int num, int size)
     }
 }
 
+static int eat_trailer(int sock, struct query *ctl)
+{
+    /* we only need this LF if we're printing ticker dots
+     * AND we are dumping protocol traces. */
+    if (outlevel >= O_VERBOSE && want_progress()) fputc('\n', stdout);
+    return (ctl->server.base_protocol->trail)(sock, ctl, tag);
+}
+
 static int fetch_messages(int mailserver_socket, struct query *ctl, 
-                         int count, int *msgsizes, int *msgcodes, int maxfetch,
-                         int *fetches, int *dispatches, int *deletions)
+                         int count, int **msgsizes, int maxfetch,
+                         int *fetches, int *dispatches, int *deletions,
+                         int *transient_errors)
 /* fetch messages in lockstep mode */
 {
-    int num, err, len;
+    flag force_retrieval;
+    int num, firstnum = 1, lastnum = 0, err, len;
+    int fetchsizelimit = ctl->fetchsizelimit;
+    int msgsize;
+    int initialfetches = *fetches;
+
+    if (ctl->server.base_protocol->getpartialsizes && NUM_NONZERO(fetchsizelimit))
+    {
+       /* for POP3, we can get the size of one mail only! Unfortunately, this
+        * protocol specific test cannot be done elsewhere as the protocol
+        * could be "auto". */
+       switch (ctl->server.protocol)
+       {
+           case P_POP3: case P_APOP: case P_RPOP:
+           fetchsizelimit = 1;
+       }
+
+       /* Time to allocate memory to store the sizes */
+       xfree(*msgsizes);
+       *msgsizes = (int *)xmalloc(sizeof(int) * fetchsizelimit);
+    }
+
+    /*
+     * What forces this code is that in POP2 and
+     * IMAP2bis you can't fetch a message without
+     * having it marked `seen'.  In POP3 and IMAP4, on the
+     * other hand, you can (peek_capable is set by 
+     * each driver module to convey this; it's not a
+     * method constant because of the difference between
+     * IMAP2bis and IMAP4, and because POP3 doesn't  peek
+     * if fetchall is on).
+     *
+     * The result of being unable to peek is that if there's
+     * any kind of transient error (DNS lookup failure, or
+     * sendmail refusing delivery due to process-table limits)
+     * the message will be marked "seen" on the server without
+     * having been delivered.  This is not a big problem if
+     * fetchmail is running in foreground, because the user
+     * will see a "skipped" message when it next runs and get
+     * clued in.
+     *
+     * But in daemon mode this leads to the message
+     * being silently ignored forever.  This is not
+     * acceptable.
+     *
+     * We compensate for this by checking the error
+     * count from the previous pass and forcing all
+     * messages to be considered new if it's nonzero.
+     */
+    force_retrieval = !peek_capable && (ctl->errcount > 0);
 
     for (num = 1; num <= count; num++)
     {
@@ -412,27 +499,84 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
        flag suppress_forward = FALSE;
        flag suppress_readbody = FALSE;
        flag retained = FALSE;
+       int msgcode = MSGLEN_UNKNOWN;
+
+       /* check if the message is old
+        * Note: the size of the message may not be known here */
+       if (ctl->fetchall || force_retrieval) {
+           /* empty */
+       } else { 
+           if (ctl->server.base_protocol->is_old && (ctl->server.base_protocol->is_old)(mailserver_socket,ctl,num)) {
+                   msgcode = MSGLEN_OLD;
+           }
+        }
+       if (msgcode == MSGLEN_OLD)
+       {
+           /*
+            * To avoid flooding the logs when using --keep, report
+            * skipping for old messages only when --flush is on.
+            */
+           if (outlevel > O_SILENT && ctl->flush)
+           {
+               report_build(stdout, 
+                            GT_("skipping message %s@%s:%d"),
+                            ctl->remotename, ctl->server.truename, num);
+           }
+
+           goto flagthemail;
+       }
+
+       if (ctl->server.base_protocol->getpartialsizes && NUM_NONZERO(fetchsizelimit) &&
+           lastnum < num)
+       {
+           /* Instead of getting the sizes of all mails at the start, we get
+            * the sizes in blocks of fetchsizelimit. This leads to better
+            * performance when there are too many mails (say, 10000) in
+            * the mailbox and either we are not getting all the mails at
+            * one go (--fetchlimit 100) or there is a frequent socket
+            * error while just getting the sizes of all mails! */
+
+           int i;
+           int oldstage = stage;
+           firstnum = num;
+           lastnum = num + fetchsizelimit - 1;
+           if (lastnum > count)
+               lastnum = count;
+           for (i = 0; i < fetchsizelimit; i++)
+               (*msgsizes)[i] = 0;
+
+           stage = STAGE_GETSIZES;
+           err = (ctl->server.base_protocol->getpartialsizes)(mailserver_socket, num, lastnum, *msgsizes);
+           if (err != 0) {
+               return err;
+           }
+           stage = oldstage;
+       }
+
+       msgsize = *msgsizes ? (*msgsizes)[num-firstnum] : 0;
 
-       if (msgcodes[num-1] < 0)
+       /* check if the message is oversized */
+       if (NUM_NONZERO(ctl->limit) && (msgsize > ctl->limit))
+           msgcode = MSGLEN_TOOLARGE;
+/*     else if (msgsize == 512)
+           msgcode = MSGLEN_OLD;  (hmh) sample code to skip message */
+
+       if (msgcode < 0)
        {
-           if ((msgcodes[num-1] == MSGLEN_TOOLARGE) && !check_only)
-               mark_oversized(ctl, num, msgsizes[num-1]);
-               /* To avoid flooding the syslog when using --keep,
-                * report "Skipped message" only when:
-                *  1) --verbose is on, or
-                *  2) fetchmail does not use syslog, or
-                *  3) the message was skipped for some other
-                *     reason than being old.
-                */
-           if (   (outlevel >= O_VERBOSE) ||
-                  (outlevel > O_SILENT && (!run.use_syslog || msgcodes[num-1] != MSGLEN_OLD))
-              )
+           if (msgcode == MSGLEN_TOOLARGE)
            {
+               mark_oversized(ctl, msgsize);
+               if (!ctl->limitflush)
+                   suppress_delete = TRUE;
+           }
+           if (outlevel > O_SILENT)
+           {
+               /* old messages are already handled above */
                report_build(stdout, 
                             GT_("skipping message %s@%s:%d (%d octets)"),
                             ctl->remotename, ctl->server.truename, num,
-                            msgsizes[num-1]);
-               switch (msgcodes[num-1])
+                            msgsize);
+               switch (msgcode)
                {
                case MSGLEN_INVALID:
                    /*
@@ -455,7 +599,11 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
        }
        else
        {
+         /* XXX FIXME: make this one variable, wholesize and
+            separatefetchbody query the same variable just with
+            inverted logic */
            flag wholesize = !ctl->server.base_protocol->fetch_body;
+           flag separatefetchbody = (ctl->server.base_protocol->fetch_body) ? TRUE : FALSE;
 
            /* request a message */
            err = (ctl->server.base_protocol->fetch_headers)(mailserver_socket,ctl,num, &len);
@@ -464,7 +612,8 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
                report(stdout,
                             GT_("couldn't fetch headers, message %s@%s:%d (%d octets)\n"),
                             ctl->remotename, ctl->server.truename, num,
-                            msgsizes[num-1]);
+                            msgsize);
+               (*transient_errors)++;
                continue;
            }
            else if (err != 0)
@@ -473,7 +622,7 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
            /* -1 means we didn't see a size in the response */
            if (len == -1)
            {
-               len = msgsizes[num - 1];
+               len = msgsize;
                wholesize = TRUE;
            }
 
@@ -484,49 +633,50 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
                             num, count);
 
                if (len > 0)
-                   report_build(stdout, GT_(" (%d %soctets)"),
-                                len, wholesize ? "" : GT_("header "));
-               if (outlevel >= O_VERBOSE)
-                   report_complete(stdout, "\n");
-               else
-                   report_complete(stdout, " ");
+                   report_build(stdout, wholesize ? GT_(" (%d octets)")
+                                : GT_(" (%d header octets)"), len);
+               if (want_progress()) {
+                   /* flush and add a blank to append ticker dots */
+                   report_flush(stdout);
+                   putchar(' ');
+               }
            }
 
            /* 
             * Read the message headers and ship them to the
             * output sink.  
             */
-           err = readheaders(mailserver_socket, len, msgsizes[num-1],
-                            ctl, num);
+           err = readheaders(mailserver_socket, len, msgsize,
+                            ctl, num,
+                            /* pass the suppress_readbody flag only if the underlying
+                             * protocol does not fetch the body separately */
+                            separatefetchbody ? 0 : &suppress_readbody);
+
            if (err == PS_RETAINED)
-           {
                suppress_forward = suppress_delete = retained = TRUE;
-               /* do not read the body only if the underlying protocol
-                * allows the body to be fetched separately */
-               if (ctl->server.base_protocol->fetch_body)
-                   suppress_readbody = TRUE;
-           }
            else if (err == PS_TRANSIENT)
            {
                suppress_delete = suppress_forward = TRUE;
-               if (ctl->server.base_protocol->fetch_body)
-                   suppress_readbody = TRUE;
+               (*transient_errors)++;
            }
            else if (err == PS_REFUSED)
-           {
                suppress_forward = TRUE;
-               if (ctl->server.base_protocol->fetch_body)
-                   suppress_readbody = TRUE;
-           }
-           else if (err == PS_TRUNCATED)
-           {
-               if (ctl->server.base_protocol->fetch_body)
-                   suppress_readbody = TRUE;
-               len = 0;        /* suppress body processing */
-           }
            else if (err)
                return(err);
 
+           /* tell server we got it OK and resynchronize */
+           if (separatefetchbody && ctl->server.base_protocol->trail)
+           {
+               err = eat_trailer(mailserver_socket, ctl);
+               if (err) return(err);
+           }
+
+           /* do not read the body which is not being forwarded only if
+            * the underlying protocol allows the body to be fetched
+            * separately */
+           if (separatefetchbody && suppress_forward)
+               suppress_readbody = TRUE;
+
            /* 
             * If we're using IMAP4 or something else that
             * can fetch headers separately from bodies,
@@ -535,19 +685,11 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
             * or other PS_REFUSED error response during
             * readheaders.
             */
-           if (ctl->server.base_protocol->fetch_body && !suppress_readbody) 
+           if (!suppress_readbody)
            {
-               if (outlevel >= O_VERBOSE && !isafile(1))
-               {
-                   fputc('\n', stdout);
-                   fflush(stdout);
-               }
-
-               if ((err = (ctl->server.base_protocol->trail)(mailserver_socket, ctl, num)))
-                   return(err);
-               len = 0;
-               if (!suppress_forward)
+               if (separatefetchbody)
                {
+                   len = -1;
                    if ((err=(ctl->server.base_protocol->fetch_body)(mailserver_socket,ctl,num,&len)))
                        return(err);
                    /*
@@ -558,44 +700,36 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
                     * string.  This violates RFC2060.
                     */
                    if (len == -1)
-                       len = msgsizes[num-1] - msgblk.msglen;
-                   if (outlevel > O_SILENT && !wholesize)
-                       report_complete(stdout,
-                                       GT_(" (%d body octets) "), len);
+                       len = msgsize - msgblk.msglen;
+                   if (!wholesize) {
+                       if (outlevel > O_SILENT)
+                           report_build(stdout,
+                                   GT_(" (%d body octets)"), len);
+                       if (want_progress()) {
+                           report_flush(stdout);
+                           putchar(' ');
+                       }
+                   }
                }
-           }
 
-           /* process the body now */
-           if (len > 0)
-           {
-               if (suppress_readbody)
-               {
-                   err = PS_SUCCESS;
-               }
-               else
-               {
-                   err = readbody(mailserver_socket,
-                                 ctl,
-                                 !suppress_forward,
-                                 len);
-               }
+               /* process the body now */
+               err = readbody(mailserver_socket,
+                             ctl,
+                             !suppress_forward,
+                             len);
+
                if (err == PS_TRANSIENT)
+               {
                    suppress_delete = suppress_forward = TRUE;
+                   (*transient_errors)++;
+               }
                else if (err)
                    return(err);
 
                /* tell server we got it OK and resynchronize */
-               if (ctl->server.base_protocol->trail)
-               {
-                   if (outlevel >= O_VERBOSE && !isafile(1))
-                   {
-                       fputc('\n', stdout);
-                       fflush(stdout);
-                   }
-
-                   err = (ctl->server.base_protocol->trail)(mailserver_socket, ctl, num);
-                   if (err != 0)
-                       return(err);
+               if (ctl->server.base_protocol->trail) {
+                   err = eat_trailer(mailserver_socket, ctl);
+                   if (err) return(err);
                }
            }
 
@@ -624,14 +758,17 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
             * It's unclear what is going on here, as the
             * QUALCOMM server (at least) seems to be
             * reporting the on-disk size correctly.
+            *
+            * qmail-pop3d also goofs up message sizes and does not
+            * count the line end characters properly.
             */
-           if (msgblk.msglen != msgsizes[num-1])
+           if (msgblk.msglen != msgsize)
            {
                if (outlevel >= O_DEBUG)
                    report(stdout,
                           GT_("message %s@%s:%d was not the expected length (%d actual != %d expected)\n"),
                           ctl->remotename, ctl->server.truename, num,
-                          msgblk.msglen, msgsizes[num-1]);
+                          msgblk.msglen, msgsize);
            }
 
            /* end-of-message processing starts here */
@@ -644,52 +781,51 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
                (*fetches)++;
        }
 
+flagthemail:
        /*
-        * At this point in flow of control, either
-        * we've bombed on a protocol error or had
-        * delivery refused by the SMTP server
-        * (unlikely -- I've never seen it) or we've
-        * seen `accepted for delivery' and the
-        * message is shipped.  It's safe to mark the
-        * message seen and delete it on the server
-        * now.
+        * At this point in flow of control,
+        * either we've bombed on a protocol error
+        * or had delivery refused by the SMTP server
+        * or we've seen `accepted for delivery' and the message is shipped.
+        * It's safe to mark the message seen and delete it on the server now.
         */
 
+       /* in softbounce mode, suppress deletion and marking as seen */
+       if (suppress_forward)
+           suppress_delete = suppress_delete || run.softbounce;
+
        /* maybe we delete this message now? */
        if (retained)
        {
            if (outlevel > O_SILENT) 
-               report(stdout, GT_(" retained\n"));
+               report_complete(stdout, GT_(" retained\n"));
        }
-       else if (ctl->server.base_protocol->delete
+       else if (ctl->server.base_protocol->delete_msg
                 && !suppress_delete
-                && ((msgcodes[num-1] >= 0) ? !ctl->keep : ctl->flush))
+                && ((msgcode >= 0 && !ctl->keep)
+                    || (msgcode == MSGLEN_OLD && ctl->flush)
+                    || (msgcode == MSGLEN_TOOLARGE && ctl->limitflush)))
        {
            (*deletions)++;
            if (outlevel > O_SILENT) 
                report_complete(stdout, GT_(" flushed\n"));
-           err = (ctl->server.base_protocol->delete)(mailserver_socket, ctl, num);
+           err = (ctl->server.base_protocol->delete_msg)(mailserver_socket, ctl, num);
            if (err != 0)
                return(err);
        }
        else
        {
-           if (   (outlevel >= O_VERBOSE) ||
-                       /* To avoid flooding the syslog when using --keep,
-                        * report "Skipped message" only when:
-                        *  1) --verbose is on, or
-                        *  2) fetchmail does not use syslog, or
-                        *  3) the message was skipped for some other
-                        *     reason than just being old.
-                        */
-                  (outlevel > O_SILENT && (!run.use_syslog || msgcodes[num-1] != MSGLEN_OLD))
-              )
+           /*
+            * To avoid flooding the logs when using --keep, report
+            * skipping of new messages only.
+            */
+           if (outlevel > O_SILENT && msgcode != MSGLEN_OLD)
            report_complete(stdout, GT_(" not flushed\n"));
 
            /* maybe we mark this message as seen now? */
            if (ctl->server.base_protocol->mark_seen
                && !suppress_delete
-               && (msgcodes[num-1] >= 0 && ctl->keep))
+               && (msgcode >= 0 && ctl->keep))
            {
                err = (ctl->server.base_protocol->mark_seen)(mailserver_socket, ctl, num);
                if (err != 0)
@@ -698,35 +834,39 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
        }
 
        /* perhaps this as many as we're ready to handle */
-       if (maxfetch && maxfetch <= *fetches && *fetches < count)
+       if (maxfetch && maxfetch <= *fetches && num < count)
        {
-           report(stdout, GT_("fetchlimit %d reached; %d messages left on server %s account %s\n"),
-                  maxfetch, count - *fetches, ctl->server.truename, ctl->remotename);
+           int remcount = count - (*fetches - initialfetches);
+           report(stdout,
+                  ngettext("fetchlimit %d reached; %d message left on server %s account %s\n",
+                           "fetchlimit %d reached; %d messages left on server %s account %s\n", remcount),
+                  maxfetch, remcount, ctl->server.truename, ctl->remotename);
            return(PS_MAXFETCH);
        }
-    }
+    } /* for (num = 1; num <= count; num++) */
 
     return(PS_SUCCESS);
 }
 
-static int do_session(ctl, proto, maxfetch)
 /* retrieve messages from server using given protocol method table */
-struct query *ctl;             /* parsed options with merged-in defaults */
-const struct method *proto;    /* protocol method table */
-const int maxfetch;            /* maximum number of messages to fetch */
+static int do_session(
+       /* parsed options with merged-in defaults */
+       struct query *ctl,
+       /* protocol method table */
+       const struct method *proto,
+       /* maximum number of messages to fetch */
+       const int maxfetch)
 {
-    int js;
-#ifdef HAVE_VOLATILE
+    static int *msgsizes;
     volatile int err, mailserver_socket = -1;  /* pacifies -Wall */
-#else
-    int err, mailserver_socket = -1;
-#endif /* HAVE_VOLATILE */
+    int tmperr;
+    int deletions = 0, js;
     const char *msg;
-    SIGHANDLERTYPE pipesave;
     SIGHANDLERTYPE alrmsave;
 
     ctl->server.base_protocol = proto;
 
+    msgsizes = NULL;
     pass = 0;
     err = 0;
     init_transact(proto);
@@ -735,35 +875,23 @@ const int maxfetch;               /* maximum number of messages to fetch */
     alrmsave = set_signal_handler(SIGALRM, timeout_handler);
     mytimeout = ctl->server.timeout;
 
-    /* set up the broken-pipe timeout */
-    pipesave = set_signal_handler(SIGPIPE, sigpipe_handler);
-
-    if ((js = setjmp(restart)))
+    if ((js = sigsetjmp(restart,1)))
     {
-#ifdef HAVE_SIGPROCMASK
-       /*
-        * Don't rely on setjmp() to restore the blocked-signal mask.
-        * It does this under BSD but is required not to under POSIX.
-        *
-        * If your Unix doesn't have sigprocmask, better hope it has
-        * BSD-like behavior.  Otherwise you may see fetchmail get
-        * permanently wedged after a second timeout on a bad read,
-        * because alarm signals were blocked after the first.
-        */
+       /* exception caught */
        sigset_t        allsigs;
 
        sigfillset(&allsigs);
        sigprocmask(SIG_UNBLOCK, &allsigs, NULL);
-#endif /* HAVE_SIGPROCMASK */
-       
-       if (js == THROW_SIGPIPE)
-       {
-           set_signal_handler(SIGPIPE, SIG_IGN);
-           report(stdout,
-                  GT_("SIGPIPE thrown from an MDA or a stream socket error\n"));
-           wait(0);
+
+       if (ai0) {
+           fm_freeaddrinfo(ai0); ai0 = NULL;
        }
-       else if (js == THROW_TIMEOUT)
+
+       if (ai1) {
+           fm_freeaddrinfo(ai1); ai1 = NULL;
+       }
+       
+       if (js == THROW_TIMEOUT)
        {
            if (phase == OPEN_WAIT)
                report(stdout,
@@ -792,16 +920,16 @@ const int maxfetch;               /* maximum number of messages to fetch */
             * timeouts just mean the frequency of mail is low.
             */
            if (timeoutcount > MAX_TIMEOUTS 
-               && !open_warning_by_mail(ctl, (struct msgblk *)NULL))
+               && !open_warning_by_mail(ctl))
            {
-               stuff_warning(ctl,
-                             GT_("Subject: fetchmail sees repeated timeouts\n"));
-               stuff_warning(ctl,
+               stuff_warning(iana_charset, ctl,
+                             GT_("Subject: fetchmail sees repeated timeouts"));
+               stuff_warning(NULL, ctl, "%s", "");
+               stuff_warning(NULL, ctl,
                              GT_("Fetchmail saw more than %d timeouts while attempting to get mail from %s@%s.\n"), 
                              MAX_TIMEOUTS,
-                             ctl->remotename,
-                             ctl->server.truename);
-               stuff_warning(ctl, 
+                             ctl->remotename, ctl->server.truename);
+               stuff_warning(NULL, ctl, 
     GT_("This could mean that your mailserver is stuck, or that your SMTP\n" \
     "server is wedged, or that your mailbox file on the server has been\n" \
     "corrupted by a server error.  You can run `fetchmail -v -v' to\n" \
@@ -817,22 +945,21 @@ const int maxfetch;               /* maximum number of messages to fetch */
     }
     else
     {
+       /* sigsetjmp returned zero -> normal operation */
        char buf[MSGBUFSIZE+1], *realhost;
-       int count, new, bytes, deletions = 0;
-       int *msgsizes = (int *)NULL;
-       int *msgcodes = (int *)NULL;
-#if INET6_ENABLE
-       int fetches, dispatches, oldphase;
-#else /* INET6_ENABLE */
-       int port, fetches, dispatches, oldphase;
-#endif /* INET6_ENABLE */
+       int count, newm, bytes;
+       int fetches, dispatches, transient_errors, oldphase;
        struct idlist *idp;
 
        /* execute pre-initialization command, if any */
        if (ctl->preconnect && (err = system(ctl->preconnect)))
        {
-           report(stderr, 
-                  GT_("pre-connection command failed with status %d\n"), err);
+           if (WIFSIGNALED(err))
+               report(stderr,
+                       GT_("pre-connection command terminated with signal %d\n"), WTERMSIG(err));
+           else
+               report(stderr,
+                       GT_("pre-connection command failed with status %d\n"), WEXITSTATUS(err));
            err = PS_SYNTAX;
            goto closeUp;
        }
@@ -841,15 +968,8 @@ const int maxfetch;                /* maximum number of messages to fetch */
        oldphase = phase;
        phase = OPEN_WAIT;
        set_timeout(mytimeout);
-#if !INET6_ENABLE
-#ifdef SSL_ENABLE
-       port = ctl->server.port ? ctl->server.port : ( ctl->use_ssl ? ctl->server.base_protocol->sslport : ctl->server.base_protocol->port );
-#else
-       port = ctl->server.port ? ctl->server.port : ctl->server.base_protocol->port;
-#endif
-#endif /* !INET6_ENABLE */
 
-#ifdef HESIOD
+#ifdef HAVE_PKG_hesiod
        /* If either the pollname or vianame are "hesiod" we want to
           lookup the user's hesiod pobox host */
        if (!strcasecmp(ctl->server.queryname, "hesiod")) {
@@ -869,7 +989,6 @@ const int maxfetch;         /* maximum number of messages to fetch */
        }
 #endif /* HESIOD */
 
-#ifdef HAVE_GETHOSTBYNAME
        /*
         * Canonicalize the server truename for later use.  This also
         * functions as a probe for whether the mailserver is accessible.
@@ -892,61 +1011,65 @@ const int maxfetch;              /* maximum number of messages to fetch */
                    goto closeUp;
                }
 
+               xfree(ctl->server.truename);
                ctl->server.truename = xstrdup(leadname);
            }
            else
            {
-               struct hostent  *namerec;
-                   
-               /* 
-                * Get the host's IP, so we can report it like this:
-                *
-                * Received: from hostname [10.0.0.1]
-                */
-               errno = 0;
-               namerec = gethostbyname(ctl->server.queryname);
-               if (namerec == (struct hostent *)NULL)
+               struct addrinfo hints, *res;
+               int error;
+
+               memset(&hints, 0, sizeof(hints));
+               hints.ai_socktype = SOCK_STREAM;
+               hints.ai_family = AF_UNSPEC;
+               hints.ai_flags = AI_CANONNAME;
+#ifdef AI_ADDRCONFIG
+               hints.ai_flags |= AI_ADDRCONFIG;
+#endif
+
+               error = fm_getaddrinfo(ctl->server.queryname, NULL, &hints, &res);
+               if (error)
                {
                    report(stderr,
-                          GT_("couldn't find canonical DNS name of %s\n"),
-                          ctl->server.pollname);
+                          GT_("couldn't find canonical DNS name of %s (%s): %s\n"),
+                          ctl->server.pollname, ctl->server.queryname,
+                          gai_strerror(error));
                    err = PS_DNS;
                    set_timeout(0);
                    phase = oldphase;
                    goto closeUp;
                }
-               else 
+               else
                {
-                   ctl->server.truename=xstrdup((char *)namerec->h_name);
-                   ctl->server.trueaddr=xmalloc(namerec->h_length);
-                   memcpy(ctl->server.trueaddr, 
-                          namerec->h_addr_list[0],
-                          namerec->h_length);
+                   xfree(ctl->server.truename);
+                   /* Older FreeBSD versions return NULL in ai_canonname
+                    * if they cannot canonicalize, rather than copying
+                    * the queryname here, as IEEE Std 1003.1-2001
+                    * requires. Work around NULL. */
+                   if (res->ai_canonname != NULL) {
+                       ctl->server.truename = xstrdup(res->ai_canonname);
+                   } else {
+                       ctl->server.truename = xstrdup(ctl->server.queryname);
+                   }
+                   ctl->server.trueaddr = (struct sockaddr *)xmalloc(res->ai_addrlen);
+                   ctl->server.trueaddr_len = res->ai_addrlen;
+                   memcpy(ctl->server.trueaddr, res->ai_addr, res->ai_addrlen);
+                   fm_freeaddrinfo(res);
                }
            }
        }
-#endif /* HAVE_GETHOSTBYNAME */
 
        realhost = ctl->server.via ? ctl->server.via : ctl->server.pollname;
 
        /* allow time for the port to be set up if we have a plugin */
        if (ctl->server.plugin)
            (void)sleep(1);
-#if INET6_ENABLE
        if ((mailserver_socket = SockOpen(realhost, 
                             ctl->server.service ? ctl->server.service : ( ctl->use_ssl ? ctl->server.base_protocol->sslservice : ctl->server.base_protocol->service ),
-                            ctl->server.netsec, ctl->server.plugin)) == -1)
-#else /* INET6_ENABLE */
-       if ((mailserver_socket = SockOpen(realhost, port, NULL, ctl->server.plugin)) == -1)
-#endif /* INET6_ENABLE */
+                            ctl->server.plugin, &ai0)) == -1)
        {
            char        errbuf[BUFSIZ];
-#if !INET6_ENABLE
            int err_no = errno;
-#ifdef HAVE_RES_SEARCH
-           if (err_no != 0 && h_errno != 0)
-               report(stderr, GT_("internal inconsistency\n"));
-#endif
            /*
             * Avoid generating a bogus error every poll cycle when we're
             * in daemon mode but the connection to the outside world
@@ -957,53 +1080,10 @@ const int maxfetch;              /* maximum number of messages to fetch */
            {
                report_build(stderr, GT_("%s connection to %s failed"), 
                             ctl->server.base_protocol->name, ctl->server.pollname);
-#ifdef HAVE_RES_SEARCH
-               if (h_errno != 0)
-               {
-                   if (h_errno == HOST_NOT_FOUND)
-                       strcpy(errbuf, GT_("host is unknown."));
-#ifndef __BEOS__
-                   else if (h_errno == NO_ADDRESS)
-                       strcpy(errbuf, GT_("name is valid but has no IP address."));
-#endif
-                   else if (h_errno == NO_RECOVERY)
-                       strcpy(errbuf, GT_("unrecoverable name server error."));
-                   else if (h_errno == TRY_AGAIN)
-                       strcpy(errbuf, GT_("temporary name server error."));
-                   else
-#ifdef HAVE_SNPRINTF
-                       snprintf(errbuf, sizeof(errbuf),
-#else
-                       sprintf(errbuf,
-#endif /* HAVE_SNPRINTF */
-                         GT_("unknown DNS error %d."), h_errno);
-               }
-               else
-#endif /* HAVE_RES_SEARCH */
-                   strcpy(errbuf, strerror(err_no));
+                   strlcpy(errbuf, strerror(err_no), sizeof(errbuf));
                report_complete(stderr, ": %s\n", errbuf);
 
-#ifdef __UNUSED
-               /* 
-                * Don't use this.  It was an attempt to address Debian bug
-                * #47143 (Notify user by mail when pop server nonexistent).
-                * Trouble is, that doesn't work; you trip over the case 
-                * where your SLIP or PPP link is down...
-                */
-               /* warn the system administrator */
-               if (open_warning_by_mail(ctl, (struct msgblk *)NULL) == 0)
-               {
-                   stuff_warning(ctl,
-                        GT_("Subject: Fetchmail unreachable-server warning.\n"
-                          "\n"
-                          "Fetchmail could not reach the mail server %s:")
-                                 ctl->server.pollname);
-                   stuff_warning(ctl, errbuf, ctl->server.pollname);
-                   close_warning_by_mail(ctl, (struct msgblk *)NULL);
-               }
-#endif
            }
-#endif /* INET6_ENABLE */
            err = PS_SOCKET;
            set_timeout(0);
            phase = oldphase;
@@ -1011,20 +1091,25 @@ const int maxfetch;             /* maximum number of messages to fetch */
        }
 
 #ifdef SSL_ENABLE
-       /* Save the socket opened. Usefull if Fetchmail hangs on SSLOpen 
-        * because the socket can be closed
+       /* Save the socket opened. Useful if Fetchmail hangs on SSLOpen 
+        * because the socket can be closed.
         */
        mailserver_socket_temp = mailserver_socket;
        set_timeout(mytimeout);
 
        /* perform initial SSL handshake on open connection */
-       /* Note:  We pass the realhost name over for certificate
-               verification.  We may want to make this configurable */
-       if (ctl->use_ssl && SSLOpen(mailserver_socket,ctl->sslcert,ctl->sslkey,ctl->sslproto,ctl->sslcertck,
-           ctl->sslcertpath,ctl->sslfingerprint,realhost,ctl->server.pollname) == -1) 
+       if (ctl->use_ssl &&
+               SSLOpen(mailserver_socket, ctl->sslcert, ctl->sslkey,
+                   ctl->sslproto, ctl->sslcertck,
+                   ctl->sslcertfile, ctl->sslcertpath,
+                   ctl->sslfingerprint, ctl->sslcommonname ?
+                   ctl->sslcommonname : realhost, ctl->server.pollname,
+                   &ctl->remotename) == -1)
        {
+           set_timeout(0);
            report(stderr, GT_("SSL connection failed.\n"));
-           goto closeUp;
+           err = PS_SOCKET;
+           goto cleanUp;
        }
        
        /* Fetchmail didn't hang on SSLOpen, 
@@ -1070,7 +1155,9 @@ const int maxfetch;               /* maximum number of messages to fetch */
        stage = STAGE_GETAUTH;
        if (ctl->server.base_protocol->getauth)
        {
+           set_timeout(mytimeout);
            err = (ctl->server.base_protocol->getauth)(mailserver_socket, ctl, buf);
+           set_timeout(0);
 
            if (err != 0)
            {
@@ -1089,6 +1176,9 @@ const int maxfetch;               /* maximum number of messages to fetch */
                           ctl->server.truename,
                           (ctl->wehaveauthed ? GT_(" (previously authorized)") : "")
                        );
+                   if (ctl->server.authenticate == A_ANY && !ctl->wehaveauthed) {
+                       report(stderr, GT_("For help, see http://www.fetchmail.info/fetchmail-FAQ.html#R15\n"));
+                   }
 
                    /*
                     * If we're running in background, try to mail the
@@ -1104,23 +1194,25 @@ const int maxfetch;             /* maximum number of messages to fetch */
                        && !ctl->wehavesentauthnote
                        && ((ctl->wehaveauthed && ++ctl->authfailcount >= 10)
                            || (!ctl->wehaveauthed && ++ctl->authfailcount >= 3))
-                       && !open_warning_by_mail(ctl, (struct msgblk *)NULL))
+                       && !open_warning_by_mail(ctl))
                    {
                        ctl->wehavesentauthnote = 1;
-                       stuff_warning(ctl,
-                                     GT_("Subject: fetchmail authentication failed on %s@%s\n"),
+                       stuff_warning(iana_charset, ctl,
+                                     GT_("Subject: fetchmail authentication failed on %s@%s"),
                            ctl->remotename, ctl->server.truename);
-                       stuff_warning(ctl,
+                       stuff_warning(NULL, ctl, "%s", "");
+                       stuff_warning(NULL, ctl,
                                      GT_("Fetchmail could not get mail from %s@%s.\n"), 
                                      ctl->remotename,
                                      ctl->server.truename);
-                       if (ctl->wehaveauthed)
-                           stuff_warning(ctl, GT_("\
+                       if (ctl->wehaveauthed) {
+                           stuff_warning(NULL, ctl, GT_("\
 The attempt to get authorization failed.\n\
 Since we have already succeeded in getting authorization for this\n\
 connection, this is probably another failure mode (such as busy server)\n\
 that fetchmail cannot distinguish because the server didn't send a useful\n\
-error message.\n\
+error message."));
+                           stuff_warning(NULL, ctl, GT_("\
 \n\
 However, if you HAVE changed your account details since starting the\n\
 fetchmail daemon, you need to stop the daemon, change your configuration\n\
@@ -1129,8 +1221,8 @@ of fetchmail, and then restart the daemon.\n\
 The fetchmail daemon will continue running and attempt to connect\n\
 at each cycle.  No future notifications will be sent until service\n\
 is restored."));
-                       else
-                           stuff_warning(ctl, GT_("\
+                       } else {
+                           stuff_warning(NULL, ctl, GT_("\
 The attempt to get authorization failed.\n\
 This probably means your password is invalid, but some servers have\n\
 other failure modes that fetchmail cannot distinguish from this\n\
@@ -1139,6 +1231,7 @@ because they don't send useful error messages on login failure.\n\
 The fetchmail daemon will continue running and attempt to connect\n\
 at each cycle.  No future notifications will be sent until service\n\
 is restored."));
+                       }
                        close_warning_by_mail(ctl, (struct msgblk *)NULL);
                    }
                }
@@ -1177,16 +1270,17 @@ is restored."));
                           GT_("Authorization OK on %s@%s\n"),
                           ctl->remotename,
                           ctl->server.truename);
-                   if (!open_warning_by_mail(ctl, (struct msgblk *)NULL))
+                   if (!open_warning_by_mail(ctl))
                    {
-                       stuff_warning(ctl,
-                             GT_("Subject: fetchmail authentication OK on %s@%s\n"),
+                       stuff_warning(iana_charset, ctl,
+                             GT_("Subject: fetchmail authentication OK on %s@%s"), 
                                      ctl->remotename, ctl->server.truename);
-                       stuff_warning(ctl,
+                       stuff_warning(NULL, ctl, "%s", "");
+                       stuff_warning(NULL, ctl,
                              GT_("Fetchmail was able to log into %s@%s.\n"), 
                                      ctl->remotename,
                                      ctl->server.truename);
-                       stuff_warning(ctl, 
+                       stuff_warning(NULL, ctl, 
                                      GT_("Service has been restored.\n"));
                        close_warning_by_mail(ctl, (struct msgblk *)NULL);
                    
@@ -1207,9 +1301,11 @@ is restored."));
        /* now iterate over each folder selected */
        for (idp = ctl->mailboxes; idp; idp = idp->next)
        {
+           ctl->folder = idp->id;
            pass = 0;
            do {
                dispatches = 0;
+               transient_errors = 0;
                ++pass;
 
                /* reset timeout, in case we did an IDLE */
@@ -1225,42 +1321,35 @@ is restored."));
 
                /* compute # of messages and number of new messages waiting */
                stage = STAGE_GETRANGE;
-               err = (ctl->server.base_protocol->getrange)(mailserver_socket, ctl, idp->id, &count, &new, &bytes);
+               err = (ctl->server.base_protocol->getrange)(mailserver_socket, ctl, idp->id, &count, &newm, &bytes);
                if (err != 0)
                    goto cleanUp;
 
                /* show user how many messages we downloaded */
                if (idp->id)
-#ifdef HAVE_SNPRINTF
                    (void) snprintf(buf, sizeof(buf),
-#else
-                   (void) sprintf(buf,
-#endif /* HAVE_SNPRINTF */
                                   GT_("%s at %s (folder %s)"),
-                                  ctl->remotename, ctl->server.truename, idp->id);
+                                  ctl->remotename, ctl->server.pollname, idp->id);
                else
-#ifdef HAVE_SNPRINTF
-                   (void) snprintf(buf, sizeof(buf),
-#else
-                   (void) sprintf(buf,
-#endif /* HAVE_SNPRINTF */
-                              GT_("%s at %s"),
-                                  ctl->remotename, ctl->server.truename);
+                   (void) snprintf(buf, sizeof(buf), GT_("%s at %s"),
+                                  ctl->remotename, ctl->server.pollname);
                if (outlevel > O_SILENT)
                {
                    if (count == -1)            /* only used for ETRN */
                        report(stdout, GT_("Polling %s\n"), ctl->server.truename);
                    else if (count != 0)
                    {
-                       if (new != -1 && (count - new) > 0)
-                           report_build(stdout, GT_("%d %s (%d seen) for %s"),
-                                 count, count > 1 ? GT_("messages") :
-                                                    GT_("message"),
-                                 count-new, buf);
+                       if (newm != -1 && (count - newm) > 0)
+                           report_build(stdout, ngettext("%d message (%d %s) for %s", "%d messages (%d %s) for %s", (unsigned long)count),
+                                 count,
+                                 count - newm, 
+                                 ngettext("seen", "seen", (unsigned long)count-newm),
+                                 buf);
                        else
-                           report_build(stdout, GT_("%d %s for %s"), 
-                                 count, count > 1 ? GT_("messages") :
-                                                    GT_("message"), buf);
+                           report_build(stdout, ngettext("%d message for %s",
+                                                         "%d messages for %s",
+                                                         count), 
+                                 count, buf);
                        if (bytes == -1)
                            report_complete(stdout, ".\n");
                        else
@@ -1280,9 +1369,9 @@ is restored."));
 
                if (check_only)
                {
-                   if (new == -1 || ctl->fetchall)
-                       new = count;
-                   fetches = new     /* set error status correctly */
+                   if (newm == -1 || ctl->fetchall)
+                       newm = count;
+                   fetches = newm;     /* set error status correctly */
                    /*
                     * There used to be a `goto noerror' here, but this
                     * prevented checking of multiple folders.  This
@@ -1292,37 +1381,7 @@ is restored."));
                }
                else if (count > 0)
                {    
-                   flag        force_retrieval;
-                   int         i, num;
-
-                   /*
-                    * What forces this code is that in POP2 and
-                    * IMAP2bis you can't fetch a message without
-                    * having it marked `seen'.  In POP3 and IMAP4, on the
-                    * other hand, you can (peek_capable is set by 
-                    * each driver module to convey this; it's not a
-                    * method constant because of the difference between
-                    * IMAP2bis and IMAP4, and because POP3 doesn't  peek
-                    * if fetchall is on).
-                    *
-                    * The result of being unable to peek is that if there's
-                    * any kind of transient error (DNS lookup failure, or
-                    * sendmail refusing delivery due to process-table limits)
-                    * the message will be marked "seen" on the server without
-                    * having been delivered.  This is not a big problem if
-                    * fetchmail is running in foreground, because the user
-                    * will see a "skipped" message when it next runs and get
-                    * clued in.
-                    *
-                    * But in daemon mode this leads to the message
-                    * being silently ignored forever.  This is not
-                    * acceptable.
-                    *
-                    * We compensate for this by checking the error
-                    * count from the previous pass and forcing all
-                    * messages to be considered new if it's nonzero.
-                    */
-                   force_retrieval = !peek_capable && (ctl->errcount > 0);
+                   int         i;
 
                    /*
                     * Don't trust the message count passed by the server.
@@ -1331,18 +1390,11 @@ is restored."));
                     * count, and allocate a malloc area that would overlap
                     * a portion of the stack.
                     */
-                   if (count > INT_MAX/sizeof(int))
+                   if ((unsigned)count > INT_MAX/sizeof(int))
                    {
                        report(stderr, GT_("bogus message count!"));
-                       return(PS_PROTOCOL);
-                   }
-
-                   /* OK, we're going to gather size info next */
-                   xalloca(msgsizes, int *, sizeof(int) * count);
-                   xalloca(msgcodes, int *, sizeof(int) * count);
-                   for (i = 0; i < count; i++) {
-                       msgsizes[i] = 0;
-                       msgcodes[i] = MSGLEN_UNKNOWN;
+                       err = PS_PROTOCOL;
+                       goto cleanUp;
                    }
 
                    /* 
@@ -1351,9 +1403,18 @@ is restored."));
                     * option.  If the protocol has a getsizes method,
                     * we presume this means it doesn't get reliable
                     * sizes from message fetch responses.
+                    *
+                    * If the protocol supports getting sizes of subset of
+                    * messages, we skip this step now.
                     */
-                   if (proto->getsizes)
+                   if (proto->getsizes &&
+                       !(proto->getpartialsizes && NUM_NONZERO(ctl->fetchsizelimit)))
                    {
+                       xfree(msgsizes);
+                       msgsizes = (int *)xmalloc(sizeof(int) * count);
+                       for (i = 0; i < count; i++)
+                           msgsizes[i] = 0;
+
                        stage = STAGE_GETSIZES;
                        err = (proto->getsizes)(mailserver_socket, count, msgsizes);
                        if (err != 0)
@@ -1367,30 +1428,27 @@ is restored."));
                        }
                    }
 
-                   /* mark some messages not to be retrieved */
-                   for (num = 1; num <= count; num++)
-                   {
-                       if (NUM_NONZERO(ctl->limit) && (msgsizes[num-1] > ctl->limit))
-                           msgcodes[num-1] = MSGLEN_TOOLARGE;
-                       else if (ctl->fetchall || force_retrieval)
-                           continue;
-                       else if (ctl->server.base_protocol->is_old && (ctl->server.base_protocol->is_old)(mailserver_socket,ctl,num))
-                           msgcodes[num-1] = MSGLEN_OLD;
-/*                     else if (msgsizes[num-1] == 512)
-                               msgcodes[num-1] = MSGLEN_OLD;  (hmh) sample code to skip message */
-                   }
-
                    /* read, forward, and delete messages */
                    stage = STAGE_FETCH;
 
                    /* fetch in lockstep mode */
                    err = fetch_messages(mailserver_socket, ctl, 
-                                        count, msgsizes, msgcodes,
+                                        count, &msgsizes,
                                         maxfetch,
-                                        &fetches, &dispatches, &deletions);
-                   if (err)
+                                        &fetches, &dispatches, &deletions,
+                                        &transient_errors);
+                   if (err != PS_SUCCESS && err != PS_MAXFETCH)
                        goto cleanUp;
 
+                   if (transient_errors > MAX_TRANSIENT_ERRORS)
+                   {
+                       if (outlevel > O_SILENT)
+                       {
+                           report(stderr, GT_("Too many mails skipped (%d > %d) due to transient errors for %s\n"),
+                                   transient_errors, MAX_TRANSIENT_ERRORS, buf);
+                       }
+                   }
+
                    if (!check_only && ctl->skipped
                        && run.poll_interval > 0 && !nodetach)
                    {
@@ -1398,24 +1456,55 @@ is restored."));
                        send_size_warnings(ctl);
                    }
                }
+
+               /* end-of-mailbox processing before we repoll or switch to another one */
+               if (ctl->server.base_protocol->end_mailbox_poll)
+               {
+                   tmperr = (ctl->server.base_protocol->end_mailbox_poll)(mailserver_socket, ctl);
+                   if (tmperr) {
+                       err = tmperr;
+                       goto cleanUp;
+                   }
+               }
+               /* Return now if we have reached the fetchlimit */
+               if (maxfetch && maxfetch <= fetches)
+                   goto no_error;
            } while
                  /*
-                  * Only re-poll if we either had some actual forwards and 
-                  * either allowed deletions and had no errors.
+                  * Only repoll if we either had some actual forwards
+                  * or are idling for new mails and had no errors.
                   * Otherwise it is far too easy to get into infinite loops.
                   */
-                 (dispatches && ctl->server.base_protocol->retry && !ctl->keep && !ctl->errcount);
+                 (ctl->server.base_protocol->retry && (dispatches || ctl->idle) && !ctl->errcount);
        }
 
-    /* no_error: */
-       /* ordinary termination with no errors -- officially log out */
-       err = (ctl->server.base_protocol->logout_cmd)(mailserver_socket, ctl);
+       /* XXX: From this point onwards, preserve err unless a new error has occurred */
+
+    no_error:
+       /* PS_SUCCESS, PS_MAXFETCH: ordinary termination with no errors -- officially log out */
+       stage = STAGE_LOGOUT;
+       tmperr = (ctl->server.base_protocol->logout_cmd)(mailserver_socket, ctl);
+       if (tmperr != PS_SUCCESS)
+           err = tmperr;
        /*
         * Hmmmm...arguably this would be incorrect if we had fetches but
         * no dispatches (due to oversized messages, etc.)
         */
-       if (err == 0)
-           err = (fetches > 0) ? PS_SUCCESS : PS_NOMAIL;
+       else if (err == PS_SUCCESS && fetches == 0)
+           err = PS_NOMAIL;
+       /*
+        * Close all SMTP delivery sockets.  For optimum performance
+        * we'd like to hold them open til end of run, but (1) this
+        * loses if our poll interval is longer than the MTA's
+        * inactivity timeout, and (2) some MTAs (like smail) don't
+        * deliver after each message, but rather queue up mail and
+        * wait to actually deliver it until the input socket is
+        * closed.
+        *
+        * don't send QUIT for ODMR case because we're acting as a
+        * proxy between the SMTP server and client.
+        */
+       smtp_close(ctl, ctl->server.protocol != P_ODMR);
        cleanupSockClose(mailserver_socket);
        goto closeUp;
 
@@ -1429,10 +1518,18 @@ is restored."));
 
        /* try to clean up all streams */
        release_sink(ctl);
+       /*
+        * Sending SMTP QUIT on signal is theoretically nice, but led
+        * to a subtle bug.  If fetchmail was terminated by signal
+        * while it was shipping message text, it would hang forever
+        * waiting for a command acknowledge.  In theory we could
+        * enable the QUIT only outside of the message send.  In
+        * practice, we don't care.  All mailservers hang up on a
+        * dropped TCP/IP connection anyway.
+        */
        smtp_close(ctl, 0);
        if (mailserver_socket != -1) {
            cleanupSockClose(mailserver_socket);
-           mailserver_socket = -1;
        }
        /* If there was a connect timeout, the socket should be closed.
         * mailserver_socket_temp contains the socket to close.
@@ -1443,7 +1540,8 @@ is restored."));
        }
     }
 
-    msg = (const char *)NULL;  /* sacrifice to -Wall */
+    /* no report on PS_AUTHFAIL */
+    msg = NULL;
     switch (err)
     {
     case PS_SOCKET:
@@ -1471,42 +1569,43 @@ is restored."));
        msg = GT_("DNS lookup");
        break;
     case PS_UNDEFINED:
-       report(stderr, GT_("undefined error\n"));
+       msg = GT_("undefined");
        break;
     }
-    /* no report on PS_MAXFETCH or PS_UNDEFINED or PS_AUTHFAIL */
-    if (err==PS_SOCKET || err==PS_SYNTAX
-               || err==PS_IOERR || err==PS_ERROR || err==PS_PROTOCOL 
-               || err==PS_LOCKBUSY || err==PS_SMTP || err==PS_DNS)
-    {
-       char    *stem;
-
-       if (phase == FORWARDING_WAIT || phase == LISTENER_WAIT)
-           stem = GT_("%s error while delivering to SMTP host %s\n");
+    if (msg) {
+       if (phase == FORWARDING_WAIT || phase == LISTENER_WAIT
+               || err == PS_SMTP)
+           report(stderr, GT_("%s error while fetching from %s@%s and delivering to SMTP host %s\n"),
+                   msg, ctl->remotename, ctl->server.pollname,
+                   ctl->smtphost ? ctl->smtphost : GT_("unknown"));
        else
-           stem = GT_("%s error while fetching from %s\n");
-       report(stderr, stem, msg, ctl->server.pollname);
+           report(stderr, GT_("%s error while fetching from %s@%s\n"),
+                   msg, ctl->remotename, ctl->server.pollname);
     }
 
 closeUp:
+    xfree(msgsizes);
+    ctl->folder = NULL;
+
     /* execute wrapup command, if any */
-    if (ctl->postconnect && (err = system(ctl->postconnect)))
+    if (ctl->postconnect && (tmperr = system(ctl->postconnect)))
     {
-       report(stderr, GT_("post-connection command failed with status %d\n"), err);
+       if (WIFSIGNALED(tmperr))
+           report(stderr, GT_("post-connection command terminated with signal %d\n"), WTERMSIG(tmperr));
+       else
+           report(stderr, GT_("post-connection command failed with status %d\n"), WEXITSTATUS(tmperr));
        if (err == PS_SUCCESS)
            err = PS_SYNTAX;
     }
 
     set_timeout(0); /* cancel any pending alarm */
     set_signal_handler(SIGALRM, alrmsave);
-    set_signal_handler(SIGPIPE, pipesave);
     return(err);
 }
 
-int do_protocol(ctl, proto)
-/* retrieve messages from server using given protocol method table */
-struct query *ctl;             /* parsed options with merged-in defaults */
-const struct method *proto;    /* protocol method table */
+/** retrieve messages from server using given protocol method table */
+int do_protocol(struct query *ctl /** parsed options with merged-in defaults */,
+               const struct method *proto /** protocol method table */)
 {
     int        err;
 
@@ -1543,7 +1642,8 @@ const struct method *proto;       /* protocol method table */
            return(PS_SYNTAX);
        }
     }
-    if (!proto->getsizes && NUM_SPECIFIED(ctl->limit))
+    if (!(proto->getsizes || proto->getpartialsizes)
+           && NUM_SPECIFIED(ctl->limit))
     {
        report(stderr,
                GT_("Option --limit is not supported with %s\n"),
@@ -1555,7 +1655,8 @@ const struct method *proto;       /* protocol method table */
      * If no expunge limit or we do expunges within the driver,
      * then just do one session, passing in any fetchlimit.
      */
-    if (proto->retry || !NUM_SPECIFIED(ctl->expunge))
+    if ((ctl->keep && !ctl->flush) ||
+       proto->retry || !NUM_SPECIFIED(ctl->expunge))
        return(do_session(ctl, proto, NUM_VALUE_OUT(ctl->fetchlimit)));
     /*
      * There's an expunge limit, and it isn't handled in the driver itself.