]> Pileus Git - ~andy/fetchmail/blobdiff - driver.c
Credit John Beck's fixes.
[~andy/fetchmail] / driver.c
index 79ad42133795b9055d052291b846e2b7c9463bd4..c2917268cedfce70fa9d70f085bad27b7f4f201b 100644 (file)
--- a/driver.c
+++ b/driver.c
 #if defined(HAVE_SYS_ITIMER_H)
 #include <sys/itimer.h>
 #endif
-#include  <sys/time.h>
 #include  <signal.h>
 #ifdef HAVE_SYS_WAIT_H
 #include <sys/wait.h>
 #endif
 
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
 #ifdef HAVE_NET_SOCKET_H
 #include <net/socket.h>
 #endif
-#ifdef HESIOD
+#include <netdb.h>
+#ifdef HAVE_PKG_hesiod
+#ifdef __cplusplus
+extern "C" {
+#endif
 #include <hesiod.h>
+#ifdef __cplusplus
+}
+#endif
 #endif
 
-#if defined(HAVE_RES_SEARCH) || defined(HAVE_GETHOSTBYNAME)
-#include <netdb.h>
-#include "mx.h"
-#endif /* defined(HAVE_RES_SEARCH) || defined(HAVE_GETHOSTBYNAME) */
+#include <langinfo.h>
 
 #include "kerberos.h"
 #ifdef KERBEROS_V4
 #include "socket.h"
 
 #include "fetchmail.h"
+#include "getaddrinfo.h"
 #include "tunable.h"
 
+#include "sdump.h"
+
 /* throw types for runtime errors */
 #define THROW_TIMEOUT  1               /* server timed out */
-#define THROW_SIGPIPE  2               /* SIGPIPE on stream socket */
 
 /* magic values for the message length array */
 #define MSGLEN_UNKNOWN 0               /* length unknown (0 is impossible) */
@@ -67,16 +75,30 @@ int stage;          /* where are we? */
 int phase;             /* where are we, for error-logging purposes? */
 int batchcount;                /* count of messages sent in current batch */
 flag peek_capable;     /* can we peek for better error recovery? */
-int mailserver_socket_temp;    /* socket to free if connect timeout */ 
+int mailserver_socket_temp = -1;       /* socket to free if connect timeout */ 
 
-static int timeoutcount;               /* count consecutive timeouts */
+struct addrinfo *ai0, *ai1;    /* clean these up after signal */
 
-static jmp_buf restart;
+static volatile int timeoutcount = 0;  /* count consecutive timeouts */
+static volatile int idletimeout = 0;   /* timeout occured in idle stage? */
+
+static sigjmp_buf      restart;
+
+int is_idletimeout(void)
+/* last timeout occured in idle stage? */
+{
+    return idletimeout;
+}
+
+void resetidletimeout(void)
+{
+    idletimeout = 0;
+}
 
 void set_timeout(int timeleft)
 /* reset the nonresponse-timeout */
 {
-#if !defined(__EMX__) && !defined(__BEOS__) 
+#if !defined(__EMX__) && !defined(__BEOS__)
     struct itimerval ntimeout;
 
     if (timeleft == 0)
@@ -89,34 +111,32 @@ void set_timeout(int timeleft)
 #endif
 }
 
-static void timeout_handler (int signal)
+static RETSIGTYPE timeout_handler (int signal)
 /* handle SIGALRM signal indicating a server timeout */
 {
-    timeoutcount++;
-    longjmp(restart, THROW_TIMEOUT);
+    (void)signal;
+    if(stage != STAGE_IDLE) {
+       timeoutcount++;
+       /* XXX FIXME: this siglongjmp must die - it's not safe to be
+        * called from a function handler and breaks, for instance,
+        * getaddrinfo() */
+       siglongjmp(restart, THROW_TIMEOUT);
+    } else
+       idletimeout = 1;
 }
 
-static void sigpipe_handler (int signal)
-/* handle SIGPIPE signal indicating a broken stream socket */
-{
-    longjmp(restart, THROW_SIGPIPE);
-}
-
-/* ignore SIGALRM signal indicating a timeout during cleanup */
-static void cleanup_timeout_handler (int signal) { }
-
 #define CLEANUP_TIMEOUT 60 /* maximum timeout during cleanup */
 
 static int cleanupSockClose (int fd)
 /* close sockets in maximum CLEANUP_TIMEOUT seconds during cleanup */
 {
     int scerror;
-    void (*alrmsave)(int);
-    alrmsave = signal(SIGALRM, cleanup_timeout_handler);
+    SIGHANDLERTYPE alrmsave;
+    alrmsave = set_signal_handler(SIGALRM, null_signal_handler);
     set_timeout(CLEANUP_TIMEOUT);
     scerror = SockClose(fd);
     set_timeout(0);
-    signal(SIGALRM, alrmsave);
+    set_signal_handler(SIGALRM, alrmsave);
     return (scerror);
 }
 
@@ -127,7 +147,6 @@ int socket;         /* socket to server host */
 char *canonical;       /* server name */
 char *principal;
 {
-    char * host_primary;
     KTEXT ticket;
     MSG_DAT msg_data;
     CREDENTIALS cred;
@@ -158,7 +177,7 @@ char *principal;
        }
     }
   
-    xalloca(ticket, KTEXT, sizeof (KTEXT_ST));
+    ticket = xmalloc(sizeof (KTEXT_ST));
     rem = (krb_sendauth (0L, socket, ticket,
                         prin ? prin : "pop",
                         inst ? inst : canonical,
@@ -170,6 +189,7 @@ char *principal;
                         ((struct sockaddr_in *) 0),
                         ((struct sockaddr_in *) 0),
                         "KPOPV0.1"));
+    free(ticket);
     if (prin_copy)
     {
         free(prin_copy);
@@ -198,22 +218,21 @@ const char *canonical;  /* server name */
     krb5_auth_context auth_context = NULL;
 
     krb5_init_context(&context);
-    krb5_init_ets(context);
     krb5_auth_con_init(context, &auth_context);
 
-    if (retval = krb5_cc_default(context, &ccdef)) {
+    if ((retval = krb5_cc_default(context, &ccdef))) {
         report(stderr, "krb5_cc_default: %s\n", error_message(retval));
         return(PS_ERROR);
     }
 
-    if (retval = krb5_cc_get_principal(context, ccdef, &client)) {
+    if ((retval = krb5_cc_get_principal(context, ccdef, &client))) {
         report(stderr, "krb5_cc_get_principal: %s\n", error_message(retval));
         return(PS_ERROR);
     }
 
-    if (retval = krb5_sname_to_principal(context, canonical, "pop",
+    if ((retval = krb5_sname_to_principal(context, canonical, "pop",
            KRB5_NT_UNKNOWN,
-           &server)) {
+           &server))) {
         report(stderr, "krb5_sname_to_principal: %s\n", error_message(retval));
         return(PS_ERROR);
     }
@@ -235,15 +254,17 @@ const char *canonical;  /* server name */
     if (retval) {
 #ifdef HEIMDAL
       if (err_ret && err_ret->e_text) {
-          report(stderr, GT_("krb5_sendauth: %s [server says '%*s'] \n"),
-                 error_message(retval),
-                 err_ret->e_text);
+         char *t = err_ret->e_text;
+         char *tt = sdump(t, strlen(t));
+          report(stderr, GT_("krb5_sendauth: %s [server says '%s']\n"),
+                 error_message(retval), tt);
+         free(tt);
 #else
       if (err_ret && err_ret->text.length) {
-          report(stderr, GT_("krb5_sendauth: %s [server says '%*s'] \n"),
-                error_message(retval),
-                err_ret->text.length,
-                err_ret->text.data);
+         char *tt = sdump(err_ret->text.data, err_ret->text.length);
+          report(stderr, GT_("krb5_sendauth: %s [server says '%s']\n"),
+                error_message(retval), tt);
+         free(tt);
 #endif
          krb5_free_error(context, err_ret);
       } else
@@ -318,14 +339,22 @@ static void send_size_warnings(struct query *ctl)
      * but it's not a disaster, either, since the skipped mail will not
      * be deleted.
      */
-    if (open_warning_by_mail(ctl, (struct msgblk *)NULL))
+    if (open_warning_by_mail(ctl))
        return;
-    stuff_warning(ctl,
-          GT_("Subject: Fetchmail oversized-messages warning.\r\n"
-            "\r\n"
-            "The following oversized messages remain on the mail server %s:"),
-                 ctl->server.pollname);
+    stuff_warning(iana_charset, ctl,
+          GT_("Subject: Fetchmail oversized-messages warning"));
+    stuff_warning(NULL, ctl, "%s", "");
+    if (ctl->limitflush)
+       stuff_warning(NULL, ctl,
+               GT_("The following oversized messages were deleted on server %s account %s:"),
+               ctl->server.pollname, ctl->remotename);
+    else
+       stuff_warning(NULL, ctl,
+               GT_("The following oversized messages remain on server %s account %s:"),
+               ctl->server.pollname, ctl->remotename);
+
+    stuff_warning(NULL, ctl, "%s", "");
+
     if (run.poll_interval == 0)
        max_warning_poll_count = 0;
     else
@@ -338,9 +367,16 @@ static void send_size_warnings(struct query *ctl)
        {
            nbr = current->val.status.mark;
            size = atoi(current->id);
-           stuff_warning(ctl, 
-                   GT_("\t%d msg %d octets long skipped by fetchmail.\r\n"),
-                   nbr, size);
+           if (ctl->limitflush)
+               stuff_warning(NULL, ctl,
+                       ngettext("  %d message  %d octets long deleted by fetchmail.",
+                                "  %d messages %d octets long deleted by fetchmail.", nbr),
+                       nbr, size);
+           else
+               stuff_warning(NULL, ctl,
+                       ngettext("  %d message  %d octets long skipped by fetchmail.",
+                                "  %d messages %d octets long skipped by fetchmail.", nbr),
+                       nbr, size);
        }
        current->val.status.num++;
        current->val.status.mark = 0;
@@ -349,10 +385,12 @@ static void send_size_warnings(struct query *ctl)
            current->val.status.num = 0;
     }
 
+    stuff_warning(NULL, ctl, "%s", "");
+
     close_warning_by_mail(ctl, (struct msgblk *)NULL);
 }
 
-static void mark_oversized(struct query *ctl, int num, int size)
+static void mark_oversized(struct query *ctl, int size)
 /* mark a message oversized */
 {
     struct idlist *current=NULL, *tmp=NULL;
@@ -360,12 +398,7 @@ static void mark_oversized(struct query *ctl, int num, int size)
     int cnt;
 
     /* convert size to string */
-#ifdef HAVE_SNPRINTF
-    snprintf(sizestr, sizeof(sizestr),
-#else
-    sprintf(sizestr,
-#endif /* HAVE_SNPRINTF */
-      "%d", size);
+    snprintf(sizestr, sizeof(sizestr), "%d", size);
 
     /* build a list of skipped messages
      * val.id = size of msg (string cnvt)
@@ -382,16 +415,9 @@ static void mark_oversized(struct query *ctl, int num, int size)
     cnt = current ? current->val.status.num : 0;
 
     /* if entry exists, increment the count */
-    if (current && str_in_list(&current, sizestr, FALSE))
+    if (current && (tmp = str_in_list(&current, sizestr, FALSE)))
     {
-       for ( ; current; current = current->next)
-       {
-           if (strcmp(current->id, sizestr) == 0)
-           {
-               current->val.status.mark++;
-               break;
-           }
-       }
+       tmp->val.status.mark++;
     }
     /* otherwise, create a new entry */
     /* initialise with current poll count */
@@ -402,12 +428,70 @@ static void mark_oversized(struct query *ctl, int num, int size)
     }
 }
 
+static int eat_trailer(int sock, struct query *ctl)
+{
+    /* we only need this LF if we're printing ticker dots
+     * AND we are dumping protocol traces. */
+    if (outlevel >= O_VERBOSE && want_progress()) fputc('\n', stdout);
+    return (ctl->server.base_protocol->trail)(sock, ctl, tag);
+}
+
 static int fetch_messages(int mailserver_socket, struct query *ctl, 
-                         int count, int *msgsizes, int *msgcodes, int maxfetch,
-                         int *fetches, int *dispatches, int *deletions)
+                         int count, int **msgsizes, int maxfetch,
+                         int *fetches, int *dispatches, int *deletions,
+                         int *transient_errors)
 /* fetch messages in lockstep mode */
 {
-    int num, err, len;
+    flag force_retrieval;
+    int num, firstnum = 1, lastnum = 0, err, len;
+    int fetchsizelimit = ctl->fetchsizelimit;
+    int msgsize;
+    int initialfetches = *fetches;
+
+    if (ctl->server.base_protocol->getpartialsizes && NUM_NONZERO(fetchsizelimit))
+    {
+       /* for POP3, we can get the size of one mail only! Unfortunately, this
+        * protocol specific test cannot be done elsewhere as the protocol
+        * could be "auto". */
+       switch (ctl->server.protocol)
+       {
+           case P_POP3: case P_APOP: case P_RPOP:
+           fetchsizelimit = 1;
+       }
+
+       /* Time to allocate memory to store the sizes */
+       xfree(*msgsizes);
+       *msgsizes = (int *)xmalloc(sizeof(int) * fetchsizelimit);
+    }
+
+    /*
+     * What forces this code is that in POP2 and
+     * IMAP2bis you can't fetch a message without
+     * having it marked `seen'.  In POP3 and IMAP4, on the
+     * other hand, you can (peek_capable is set by 
+     * each driver module to convey this; it's not a
+     * method constant because of the difference between
+     * IMAP2bis and IMAP4, and because POP3 doesn't  peek
+     * if fetchall is on).
+     *
+     * The result of being unable to peek is that if there's
+     * any kind of transient error (DNS lookup failure, or
+     * sendmail refusing delivery due to process-table limits)
+     * the message will be marked "seen" on the server without
+     * having been delivered.  This is not a big problem if
+     * fetchmail is running in foreground, because the user
+     * will see a "skipped" message when it next runs and get
+     * clued in.
+     *
+     * But in daemon mode this leads to the message
+     * being silently ignored forever.  This is not
+     * acceptable.
+     *
+     * We compensate for this by checking the error
+     * count from the previous pass and forcing all
+     * messages to be considered new if it's nonzero.
+     */
+    force_retrieval = !peek_capable && (ctl->errcount > 0);
 
     for (num = 1; num <= count; num++)
     {
@@ -415,18 +499,84 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
        flag suppress_forward = FALSE;
        flag suppress_readbody = FALSE;
        flag retained = FALSE;
+       int msgcode = MSGLEN_UNKNOWN;
+
+       /* check if the message is old
+        * Note: the size of the message may not be known here */
+       if (ctl->fetchall || force_retrieval) {
+           /* empty */
+       } else { 
+           if (ctl->server.base_protocol->is_old && (ctl->server.base_protocol->is_old)(mailserver_socket,ctl,num)) {
+                   msgcode = MSGLEN_OLD;
+           }
+        }
+       if (msgcode == MSGLEN_OLD)
+       {
+           /*
+            * To avoid flooding the logs when using --keep, report
+            * skipping for old messages only when --flush is on.
+            */
+           if (outlevel > O_SILENT && ctl->flush)
+           {
+               report_build(stdout, 
+                            GT_("skipping message %s@%s:%d"),
+                            ctl->remotename, ctl->server.truename, num);
+           }
+
+           goto flagthemail;
+       }
+
+       if (ctl->server.base_protocol->getpartialsizes && NUM_NONZERO(fetchsizelimit) &&
+           lastnum < num)
+       {
+           /* Instead of getting the sizes of all mails at the start, we get
+            * the sizes in blocks of fetchsizelimit. This leads to better
+            * performance when there are too many mails (say, 10000) in
+            * the mailbox and either we are not getting all the mails at
+            * one go (--fetchlimit 100) or there is a frequent socket
+            * error while just getting the sizes of all mails! */
+
+           int i;
+           int oldstage = stage;
+           firstnum = num;
+           lastnum = num + fetchsizelimit - 1;
+           if (lastnum > count)
+               lastnum = count;
+           for (i = 0; i < fetchsizelimit; i++)
+               (*msgsizes)[i] = 0;
+
+           stage = STAGE_GETSIZES;
+           err = (ctl->server.base_protocol->getpartialsizes)(mailserver_socket, num, lastnum, *msgsizes);
+           if (err != 0) {
+               return err;
+           }
+           stage = oldstage;
+       }
+
+       msgsize = *msgsizes ? (*msgsizes)[num-firstnum] : 0;
+
+       /* check if the message is oversized */
+       if (NUM_NONZERO(ctl->limit) && (msgsize > ctl->limit))
+           msgcode = MSGLEN_TOOLARGE;
+/*     else if (msgsize == 512)
+           msgcode = MSGLEN_OLD;  (hmh) sample code to skip message */
 
-       if (msgcodes[num-1] < 0)
+       if (msgcode < 0)
        {
-           if ((msgcodes[num-1] == MSGLEN_TOOLARGE) && !check_only)
-               mark_oversized(ctl, num, msgsizes[num-1]);
+           if (msgcode == MSGLEN_TOOLARGE)
+           {
+               mark_oversized(ctl, msgsize);
+               if (!ctl->limitflush)
+                   suppress_delete = TRUE;
+           }
            if (outlevel > O_SILENT)
            {
+               /* old messages are already handled above */
                report_build(stdout, 
                             GT_("skipping message %s@%s:%d (%d octets)"),
                             ctl->remotename, ctl->server.truename, num,
-                            msgsizes[num-1]);
-               switch (msgcodes[num-1])
+                            msgsize);
+               switch (msgcode)
                {
                case MSGLEN_INVALID:
                    /*
@@ -442,25 +592,28 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
                    report_build(stdout, GT_(" (length -1)"));
                    break;
                case MSGLEN_TOOLARGE:
-                   report_build(stdout, 
-                                GT_(" (oversized, %d octets)"),
-                                msgsizes[num-1]);
+                   report_build(stdout, GT_(" (oversized)"));
                    break;
                }
            }
        }
        else
        {
+         /* XXX FIXME: make this one variable, wholesize and
+            separatefetchbody query the same variable just with
+            inverted logic */
            flag wholesize = !ctl->server.base_protocol->fetch_body;
+           flag separatefetchbody = (ctl->server.base_protocol->fetch_body) ? TRUE : FALSE;
 
            /* request a message */
            err = (ctl->server.base_protocol->fetch_headers)(mailserver_socket,ctl,num, &len);
            if (err == PS_TRANSIENT)    /* server is probably Exchange */
            {
-               report_build(stdout,
-                            GT_("couldn't fetch headers, message %s@%s:%d (%d octets)"),
+               report(stdout,
+                            GT_("couldn't fetch headers, message %s@%s:%d (%d octets)\n"),
                             ctl->remotename, ctl->server.truename, num,
-                            msgsizes[num-1]);
+                            msgsize);
+               (*transient_errors)++;
                continue;
            }
            else if (err != 0)
@@ -469,7 +622,7 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
            /* -1 means we didn't see a size in the response */
            if (len == -1)
            {
-               len = msgsizes[num - 1];
+               len = msgsize;
                wholesize = TRUE;
            }
 
@@ -480,43 +633,49 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
                             num, count);
 
                if (len > 0)
-                   report_build(stdout, GT_(" (%d %soctets)"),
-                                len, wholesize ? "" : GT_("header "));
-               if (outlevel >= O_VERBOSE)
-                   report_complete(stdout, "\n");
-               else
-                   report_complete(stdout, " ");
+                   report_build(stdout, wholesize ? GT_(" (%d octets)")
+                                : GT_(" (%d header octets)"), len);
+               if (want_progress()) {
+                   /* flush and add a blank to append ticker dots */
+                   report_flush(stdout);
+                   putchar(' ');
+               }
            }
 
            /* 
             * Read the message headers and ship them to the
             * output sink.  
             */
-           err = readheaders(mailserver_socket, len, msgsizes[num-1],
-                            ctl, num);
+           err = readheaders(mailserver_socket, len, msgsize,
+                            ctl, num,
+                            /* pass the suppress_readbody flag only if the underlying
+                             * protocol does not fetch the body separately */
+                            separatefetchbody ? 0 : &suppress_readbody);
+
            if (err == PS_RETAINED)
-               suppress_readbody = suppress_forward = suppress_delete = retained = TRUE;
+               suppress_forward = suppress_delete = retained = TRUE;
            else if (err == PS_TRANSIENT)
+           {
                suppress_delete = suppress_forward = TRUE;
+               (*transient_errors)++;
+           }
            else if (err == PS_REFUSED)
                suppress_forward = TRUE;
-#if 0
-           /* 
-            * readheaders does not read the body when it
-            * hits a non-header. It has been recently
-            * fixed to return PS_TRUNCATED (properly) when
-            * that happens, but apparently fixing that bug
-            * opened this one here (which looks like an 
-            * inproper fix from some ancient thinko)
-            */
-           else if (err == PS_TRUNCATED)
-               suppress_readbody = TRUE;
            else if (err)
                return(err);
-#else
-           else if (err && err != PS_TRUNCATED)
-               return(err);
-#endif
+
+           /* tell server we got it OK and resynchronize */
+           if (separatefetchbody && ctl->server.base_protocol->trail)
+           {
+               err = eat_trailer(mailserver_socket, ctl);
+               if (err) return(err);
+           }
+
+           /* do not read the body which is not being forwarded only if
+            * the underlying protocol allows the body to be fetched
+            * separately */
+           if (separatefetchbody && suppress_forward)
+               suppress_readbody = TRUE;
 
            /* 
             * If we're using IMAP4 or something else that
@@ -526,19 +685,11 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
             * or other PS_REFUSED error response during
             * readheaders.
             */
-           if (ctl->server.base_protocol->fetch_body && !suppress_readbody) 
+           if (!suppress_readbody)
            {
-               if (outlevel >= O_VERBOSE && !isafile(1))
-               {
-                   fputc('\n', stdout);
-                   fflush(stdout);
-               }
-
-               if ((err = (ctl->server.base_protocol->trail)(mailserver_socket, ctl, num)))
-                   return(err);
-               len = 0;
-               if (!suppress_forward)
+               if (separatefetchbody)
                {
+                   len = -1;
                    if ((err=(ctl->server.base_protocol->fetch_body)(mailserver_socket,ctl,num,&len)))
                        return(err);
                    /*
@@ -549,44 +700,36 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
                     * string.  This violates RFC2060.
                     */
                    if (len == -1)
-                       len = msgsizes[num-1] - msgblk.msglen;
-                   if (outlevel > O_SILENT && !wholesize)
-                       report_complete(stdout,
-                                       GT_(" (%d body octets) "), len);
+                       len = msgsize - msgblk.msglen;
+                   if (!wholesize) {
+                       if (outlevel > O_SILENT)
+                           report_build(stdout,
+                                   GT_(" (%d body octets)"), len);
+                       if (want_progress()) {
+                           report_flush(stdout);
+                           putchar(' ');
+                       }
+                   }
                }
-           }
 
-           /* process the body now */
-           if (len > 0)
-           {
-               if (suppress_readbody)
-               {
-                   err = PS_SUCCESS;
-               }
-               else
-               {
-                   err = readbody(mailserver_socket,
-                                 ctl,
-                                 !suppress_forward,
-                                 len);
-               }
+               /* process the body now */
+               err = readbody(mailserver_socket,
+                             ctl,
+                             !suppress_forward,
+                             len);
+
                if (err == PS_TRANSIENT)
+               {
                    suppress_delete = suppress_forward = TRUE;
+                   (*transient_errors)++;
+               }
                else if (err)
                    return(err);
 
                /* tell server we got it OK and resynchronize */
-               if (ctl->server.base_protocol->trail)
-               {
-                   if (outlevel >= O_VERBOSE && !isafile(1))
-                   {
-                       fputc('\n', stdout);
-                       fflush(stdout);
-                   }
-
-                   err = (ctl->server.base_protocol->trail)(mailserver_socket, ctl, num);
-                   if (err != 0)
-                       return(err);
+               if (ctl->server.base_protocol->trail) {
+                   err = eat_trailer(mailserver_socket, ctl);
+                   if (err) return(err);
                }
            }
 
@@ -615,14 +758,17 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
             * It's unclear what is going on here, as the
             * QUALCOMM server (at least) seems to be
             * reporting the on-disk size correctly.
+            *
+            * qmail-pop3d also goofs up message sizes and does not
+            * count the line end characters properly.
             */
-           if (msgblk.msglen != msgsizes[num-1])
+           if (msgblk.msglen != msgsize)
            {
                if (outlevel >= O_DEBUG)
                    report(stdout,
                           GT_("message %s@%s:%d was not the expected length (%d actual != %d expected)\n"),
                           ctl->remotename, ctl->server.truename, num,
-                          msgblk.msglen, msgsizes[num-1]);
+                          msgblk.msglen, msgsize);
            }
 
            /* end-of-message processing starts here */
@@ -635,133 +781,117 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
                (*fetches)++;
        }
 
+flagthemail:
        /*
-        * At this point in flow of control, either
-        * we've bombed on a protocol error or had
-        * delivery refused by the SMTP server
-        * (unlikely -- I've never seen it) or we've
-        * seen `accepted for delivery' and the
-        * message is shipped.  It's safe to mark the
-        * message seen and delete it on the server
-        * now.
+        * At this point in flow of control,
+        * either we've bombed on a protocol error
+        * or had delivery refused by the SMTP server
+        * or we've seen `accepted for delivery' and the message is shipped.
+        * It's safe to mark the message seen and delete it on the server now.
         */
 
-       /*
-        * Tell the UID code we've seen this.
-        * Matthias Andree: only register the UID if we could actually
-        * forward this mail. If we omit this !suppress_delete check,
-        * fetchmail will never retry mail that the local listener
-        * refused temporarily.
-        */
-       if (ctl->newsaved && !suppress_delete)
-       {
-           struct idlist       *sdp;
-
-           for (sdp = ctl->newsaved; sdp; sdp = sdp->next)
-               if ((sdp->val.status.num == num) && (msgcodes[num-1] >= 0))
-               {
-                   sdp->val.status.mark = UID_SEEN;
-                   save_str(&ctl->oldsaved, sdp->id,UID_SEEN);
-               }
-       }
+       /* in softbounce mode, suppress deletion and marking as seen */
+       if (suppress_forward)
+           suppress_delete = suppress_delete || run.softbounce;
 
        /* maybe we delete this message now? */
        if (retained)
        {
            if (outlevel > O_SILENT) 
-               report(stdout, GT_(" retained\n"));
+               report_complete(stdout, GT_(" retained\n"));
        }
-       else if (ctl->server.base_protocol->delete
+       else if (ctl->server.base_protocol->delete_msg
                 && !suppress_delete
-                && ((msgcodes[num-1] >= 0) ? !ctl->keep : ctl->flush))
+                && ((msgcode >= 0 && !ctl->keep)
+                    || (msgcode == MSGLEN_OLD && ctl->flush)
+                    || (msgcode == MSGLEN_TOOLARGE && ctl->limitflush)))
        {
            (*deletions)++;
            if (outlevel > O_SILENT) 
                report_complete(stdout, GT_(" flushed\n"));
-           err = (ctl->server.base_protocol->delete)(mailserver_socket, ctl, num);
+           err = (ctl->server.base_protocol->delete_msg)(mailserver_socket, ctl, num);
            if (err != 0)
                return(err);
-#ifdef POP3_ENABLE
-           delete_str(&ctl->newsaved, num);
-#endif /* POP3_ENABLE */
        }
-       else if (outlevel > O_SILENT)
+       else
+       {
+           /*
+            * To avoid flooding the logs when using --keep, report
+            * skipping of new messages only.
+            */
+           if (outlevel > O_SILENT && msgcode != MSGLEN_OLD)
            report_complete(stdout, GT_(" not flushed\n"));
 
+           /* maybe we mark this message as seen now? */
+           if (ctl->server.base_protocol->mark_seen
+               && !suppress_delete
+               && (msgcode >= 0 && ctl->keep))
+           {
+               err = (ctl->server.base_protocol->mark_seen)(mailserver_socket, ctl, num);
+               if (err != 0)
+                   return(err);
+           }
+       }
+
        /* perhaps this as many as we're ready to handle */
-       if (maxfetch && maxfetch <= *fetches && *fetches < count)
+       if (maxfetch && maxfetch <= *fetches && num < count)
        {
-           report(stdout, GT_("fetchlimit %d reached; %d messages left on server %s account %s\n"),
-                  maxfetch, count - *fetches, ctl->server.truename, ctl->remotename);
+           int remcount = count - (*fetches - initialfetches);
+           report(stdout,
+                  ngettext("fetchlimit %d reached; %d message left on server %s account %s\n",
+                           "fetchlimit %d reached; %d messages left on server %s account %s\n", remcount),
+                  maxfetch, remcount, ctl->server.truename, ctl->remotename);
            return(PS_MAXFETCH);
        }
-    }
+    } /* for (num = 1; num <= count; num++) */
 
     return(PS_SUCCESS);
 }
 
-static int do_session(ctl, proto, maxfetch)
 /* retrieve messages from server using given protocol method table */
-struct query *ctl;             /* parsed options with merged-in defaults */
-const struct method *proto;    /* protocol method table */
-const int maxfetch;            /* maximum number of messages to fetch */
+static int do_session(
+       /* parsed options with merged-in defaults */
+       struct query *ctl,
+       /* protocol method table */
+       const struct method *proto,
+       /* maximum number of messages to fetch */
+       const int maxfetch)
 {
-    int js;
-#ifdef HAVE_VOLATILE
+    static int *msgsizes;
     volatile int err, mailserver_socket = -1;  /* pacifies -Wall */
-#else
-    int err, mailserver_socket = -1;
-#endif /* HAVE_VOLATILE */
+    int tmperr;
+    int deletions = 0, js;
     const char *msg;
-    void (*pipesave)(int);
-    void (*alrmsave)(int);
+    SIGHANDLERTYPE alrmsave;
 
     ctl->server.base_protocol = proto;
 
+    msgsizes = NULL;
     pass = 0;
     err = 0;
     init_transact(proto);
 
     /* set up the server-nonresponse timeout */
-    alrmsave = signal(SIGALRM, timeout_handler);
+    alrmsave = set_signal_handler(SIGALRM, timeout_handler);
     mytimeout = ctl->server.timeout;
 
-    /* set up the broken-pipe timeout */
-    pipesave = signal(SIGPIPE, sigpipe_handler);
-
-    if ((js = setjmp(restart)))
+    if ((js = sigsetjmp(restart,1)))
     {
-#ifdef HAVE_SIGPROCMASK
-       /*
-        * Don't rely on setjmp() to restore the blocked-signal mask.
-        * It does this under BSD but is required not to under POSIX.
-        *
-        * If your Unix doesn't have sigprocmask, better hope it has
-        * BSD-like behavior.  Otherwise you may see fetchmail get
-        * permanently wedged after a second timeout on a bad read,
-        * because alarm signals were blocked after the first.
-        */
+       /* exception caught */
        sigset_t        allsigs;
 
        sigfillset(&allsigs);
        sigprocmask(SIG_UNBLOCK, &allsigs, NULL);
-#endif /* HAVE_SIGPROCMASK */
-       
-       /* If there was a connect timeout, the socket should be closed.
-        * mailserver_socket_temp contains the socket to close.
-        */
-       mailserver_socket = mailserver_socket_temp;
-       
-       if (js == THROW_SIGPIPE)
-       {
-           signal(SIGPIPE, SIG_IGN);
-           report(stdout,
-                  GT_("SIGPIPE thrown from an MDA or a stream socket error\n"));
-           wait(0);
-           err = PS_SOCKET;
-           goto cleanUp;
+
+       if (ai0) {
+           fm_freeaddrinfo(ai0); ai0 = NULL;
+       }
+
+       if (ai1) {
+           fm_freeaddrinfo(ai1); ai1 = NULL;
        }
-       else if (js == THROW_TIMEOUT)
+       
+       if (js == THROW_TIMEOUT)
        {
            if (phase == OPEN_WAIT)
                report(stdout,
@@ -790,54 +920,46 @@ const int maxfetch;               /* maximum number of messages to fetch */
             * timeouts just mean the frequency of mail is low.
             */
            if (timeoutcount > MAX_TIMEOUTS 
-               && !open_warning_by_mail(ctl, (struct msgblk *)NULL))
+               && !open_warning_by_mail(ctl))
            {
-               stuff_warning(ctl,
-                             GT_("Subject: fetchmail sees repeated timeouts\r\n"));
-               stuff_warning(ctl,
-                             GT_("Fetchmail saw more than %d timeouts while attempting to get mail from %s@%s.\r\n"), 
+               stuff_warning(iana_charset, ctl,
+                             GT_("Subject: fetchmail sees repeated timeouts"));
+               stuff_warning(NULL, ctl, "%s", "");
+               stuff_warning(NULL, ctl,
+                             GT_("Fetchmail saw more than %d timeouts while attempting to get mail from %s@%s.\n"), 
                              MAX_TIMEOUTS,
-                             ctl->remotename,
-                             ctl->server.truename);
-               stuff_warning(ctl, 
-    GT_("This could mean that your mailserver is stuck, or that your SMTP\r\n" \
-    "server is wedged, or that your mailbox file on the server has been\r\n" \
-    "corrupted by a server error.  You can run `fetchmail -v -v' to\r\n" \
-    "diagnose the problem.\r\n\r\n" \
-    "Fetchmail won't poll this mailbox again until you restart it.\r\n"));
+                             ctl->remotename, ctl->server.truename);
+               stuff_warning(NULL, ctl, 
+    GT_("This could mean that your mailserver is stuck, or that your SMTP\n" \
+    "server is wedged, or that your mailbox file on the server has been\n" \
+    "corrupted by a server error.  You can run `fetchmail -v -v' to\n" \
+    "diagnose the problem.\n\n" \
+    "Fetchmail won't poll this mailbox again until you restart it.\n"));
                close_warning_by_mail(ctl, (struct msgblk *)NULL);
                ctl->wedged = TRUE;
            }
-
-           err = PS_ERROR;
        }
 
-       /* try to clean up all streams */
-       release_sink(ctl);
-       smtp_close(ctl, 0);
-       if (mailserver_socket != -1) {
-           cleanupSockClose(mailserver_socket);
-           mailserver_socket = -1;
-       }
+       err = PS_SOCKET;
+       goto cleanUp;
     }
     else
     {
+       /* sigsetjmp returned zero -> normal operation */
        char buf[MSGBUFSIZE+1], *realhost;
-       int count, new, bytes, deletions = 0;
-       int *msgsizes = (int *)NULL;
-       int *msgcodes = (int *)NULL;
-#if INET6_ENABLE
-       int fetches, dispatches, oldphase;
-#else /* INET6_ENABLE */
-       int port, fetches, dispatches, oldphase;
-#endif /* INET6_ENABLE */
+       int count, newm, bytes;
+       int fetches, dispatches, transient_errors, oldphase;
        struct idlist *idp;
 
        /* execute pre-initialization command, if any */
        if (ctl->preconnect && (err = system(ctl->preconnect)))
        {
-           report(stderr, 
-                  GT_("pre-connection command failed with status %d\n"), err);
+           if (WIFSIGNALED(err))
+               report(stderr,
+                       GT_("pre-connection command terminated with signal %d\n"), WTERMSIG(err));
+           else
+               report(stderr,
+                       GT_("pre-connection command failed with status %d\n"), WEXITSTATUS(err));
            err = PS_SYNTAX;
            goto closeUp;
        }
@@ -846,15 +968,8 @@ const int maxfetch;                /* maximum number of messages to fetch */
        oldphase = phase;
        phase = OPEN_WAIT;
        set_timeout(mytimeout);
-#if !INET6_ENABLE
-#ifdef SSL_ENABLE
-       port = ctl->server.port ? ctl->server.port : ( ctl->use_ssl ? ctl->server.base_protocol->sslport : ctl->server.base_protocol->port );
-#else
-       port = ctl->server.port ? ctl->server.port : ctl->server.base_protocol->port;
-#endif
-#endif /* !INET6_ENABLE */
 
-#ifdef HESIOD
+#ifdef HAVE_PKG_hesiod
        /* If either the pollname or vianame are "hesiod" we want to
           lookup the user's hesiod pobox host */
        if (!strcasecmp(ctl->server.queryname, "hesiod")) {
@@ -874,7 +989,6 @@ const int maxfetch;         /* maximum number of messages to fetch */
        }
 #endif /* HESIOD */
 
-#ifdef HAVE_GETHOSTBYNAME
        /*
         * Canonicalize the server truename for later use.  This also
         * functions as a probe for whether the mailserver is accessible.
@@ -897,61 +1011,65 @@ const int maxfetch;              /* maximum number of messages to fetch */
                    goto closeUp;
                }
 
+               xfree(ctl->server.truename);
                ctl->server.truename = xstrdup(leadname);
            }
            else
            {
-               struct hostent  *namerec;
-                   
-               /* 
-                * Get the host's IP, so we can report it like this:
-                *
-                * Received: from hostname [10.0.0.1]
-                */
-               errno = 0;
-               namerec = gethostbyname(ctl->server.queryname);
-               if (namerec == (struct hostent *)NULL)
+               struct addrinfo hints, *res;
+               int error;
+
+               memset(&hints, 0, sizeof(hints));
+               hints.ai_socktype = SOCK_STREAM;
+               hints.ai_family = AF_UNSPEC;
+               hints.ai_flags = AI_CANONNAME;
+#ifdef AI_ADDRCONFIG
+               hints.ai_flags |= AI_ADDRCONFIG;
+#endif
+
+               error = fm_getaddrinfo(ctl->server.queryname, NULL, &hints, &res);
+               if (error)
                {
                    report(stderr,
-                          GT_("couldn't find canonical DNS name of %s\n"),
-                          ctl->server.pollname);
+                          GT_("couldn't find canonical DNS name of %s (%s): %s\n"),
+                          ctl->server.pollname, ctl->server.queryname,
+                          gai_strerror(error));
                    err = PS_DNS;
                    set_timeout(0);
                    phase = oldphase;
                    goto closeUp;
                }
-               else 
+               else
                {
-                   ctl->server.truename=xstrdup((char *)namerec->h_name);
-                   ctl->server.trueaddr=xmalloc(namerec->h_length);
-                   memcpy(ctl->server.trueaddr, 
-                          namerec->h_addr_list[0],
-                          namerec->h_length);
+                   xfree(ctl->server.truename);
+                   /* Older FreeBSD versions return NULL in ai_canonname
+                    * if they cannot canonicalize, rather than copying
+                    * the queryname here, as IEEE Std 1003.1-2001
+                    * requires. Work around NULL. */
+                   if (res->ai_canonname != NULL) {
+                       ctl->server.truename = xstrdup(res->ai_canonname);
+                   } else {
+                       ctl->server.truename = xstrdup(ctl->server.queryname);
+                   }
+                   ctl->server.trueaddr = (struct sockaddr *)xmalloc(res->ai_addrlen);
+                   ctl->server.trueaddr_len = res->ai_addrlen;
+                   memcpy(ctl->server.trueaddr, res->ai_addr, res->ai_addrlen);
+                   fm_freeaddrinfo(res);
                }
            }
        }
-#endif /* HAVE_GETHOSTBYNAME */
 
        realhost = ctl->server.via ? ctl->server.via : ctl->server.pollname;
 
        /* allow time for the port to be set up if we have a plugin */
        if (ctl->server.plugin)
            (void)sleep(1);
-#if INET6_ENABLE
        if ((mailserver_socket = SockOpen(realhost, 
                             ctl->server.service ? ctl->server.service : ( ctl->use_ssl ? ctl->server.base_protocol->sslservice : ctl->server.base_protocol->service ),
-                            ctl->server.netsec, ctl->server.plugin)) == -1)
-#else /* INET6_ENABLE */
-       if ((mailserver_socket = SockOpen(realhost, port, NULL, ctl->server.plugin)) == -1)
-#endif /* INET6_ENABLE */
+                            ctl->server.plugin, &ai0)) == -1)
        {
            char        errbuf[BUFSIZ];
-#if !INET6_ENABLE
            int err_no = errno;
-#ifdef HAVE_RES_SEARCH
-           if (err_no != 0 && h_errno != 0)
-               report(stderr, GT_("internal inconsistency\n"));
-#endif
            /*
             * Avoid generating a bogus error every poll cycle when we're
             * in daemon mode but the connection to the outside world
@@ -962,53 +1080,10 @@ const int maxfetch;              /* maximum number of messages to fetch */
            {
                report_build(stderr, GT_("%s connection to %s failed"), 
                             ctl->server.base_protocol->name, ctl->server.pollname);
-#ifdef HAVE_RES_SEARCH
-               if (h_errno != 0)
-               {
-                   if (h_errno == HOST_NOT_FOUND)
-                       strcpy(errbuf, GT_("host is unknown."));
-#ifndef __BEOS__
-                   else if (h_errno == NO_ADDRESS)
-                       strcpy(errbuf, GT_("name is valid but has no IP address."));
-#endif
-                   else if (h_errno == NO_RECOVERY)
-                       strcpy(errbuf, GT_("unrecoverable name server error."));
-                   else if (h_errno == TRY_AGAIN)
-                       strcpy(errbuf, GT_("temporary name server error."));
-                   else
-#ifdef HAVE_SNPRINTF
-                       snprintf(errbuf, sizeof(errbuf),
-#else
-                       sprintf(errbuf,
-#endif /* HAVE_SNPRINTF */
-                         GT_("unknown DNS error %d."), h_errno);
-               }
-               else
-#endif /* HAVE_RES_SEARCH */
-                   strcpy(errbuf, strerror(err_no));
+                   strlcpy(errbuf, strerror(err_no), sizeof(errbuf));
                report_complete(stderr, ": %s\n", errbuf);
 
-#ifdef __UNUSED
-               /* 
-                * Don't use this.  It was an attempt to address Debian bug
-                * #47143 (Notify user by mail when pop server nonexistent).
-                * Trouble is, that doesn't work; you trip over the case 
-                * where your SLIP or PPP link is down...
-                */
-               /* warn the system administrator */
-               if (open_warning_by_mail(ctl, (struct msgblk *)NULL) == 0)
-               {
-                   stuff_warning(ctl,
-                        GT_("Subject: Fetchmail unreachable-server warning.\r\n"
-                          "\r\n"
-                          "Fetchmail could not reach the mail server %s:")
-                                 ctl->server.pollname);
-                   stuff_warning(ctl, errbuf, ctl->server.pollname);
-                   close_warning_by_mail(ctl, (struct msgblk *)NULL);
-               }
-#endif
            }
-#endif /* INET6_ENABLE */
            err = PS_SOCKET;
            set_timeout(0);
            phase = oldphase;
@@ -1016,20 +1091,25 @@ const int maxfetch;             /* maximum number of messages to fetch */
        }
 
 #ifdef SSL_ENABLE
-       /* Save the socket opened. Usefull if Fetchmail hangs on SSLOpen 
-        * because the socket can be closed
+       /* Save the socket opened. Useful if Fetchmail hangs on SSLOpen 
+        * because the socket can be closed.
         */
        mailserver_socket_temp = mailserver_socket;
        set_timeout(mytimeout);
 
        /* perform initial SSL handshake on open connection */
-       /* Note:  We pass the realhost name over for certificate
-               verification.  We may want to make this configurable */
-       if (ctl->use_ssl && SSLOpen(mailserver_socket,ctl->sslcert,ctl->sslkey,ctl->sslproto,ctl->sslcertck,
-           ctl->sslcertpath,ctl->sslfingerprint,realhost,ctl->server.pollname) == -1) 
+       if (ctl->use_ssl &&
+               SSLOpen(mailserver_socket, ctl->sslcert, ctl->sslkey,
+                   ctl->sslproto, ctl->sslcertck,
+                   ctl->sslcertfile, ctl->sslcertpath,
+                   ctl->sslfingerprint, ctl->sslcommonname ?
+                   ctl->sslcommonname : realhost, ctl->server.pollname,
+                   &ctl->remotename) == -1)
        {
+           set_timeout(0);
            report(stderr, GT_("SSL connection failed.\n"));
-           goto closeUp;
+           err = PS_SOCKET;
+           goto cleanUp;
        }
        
        /* Fetchmail didn't hang on SSLOpen, 
@@ -1075,7 +1155,9 @@ const int maxfetch;               /* maximum number of messages to fetch */
        stage = STAGE_GETAUTH;
        if (ctl->server.base_protocol->getauth)
        {
+           set_timeout(mytimeout);
            err = (ctl->server.base_protocol->getauth)(mailserver_socket, ctl, buf);
+           set_timeout(0);
 
            if (err != 0)
            {
@@ -1094,6 +1176,9 @@ const int maxfetch;               /* maximum number of messages to fetch */
                           ctl->server.truename,
                           (ctl->wehaveauthed ? GT_(" (previously authorized)") : "")
                        );
+                   if (ctl->server.authenticate == A_ANY && !ctl->wehaveauthed) {
+                       report(stderr, GT_("For help, see http://www.fetchmail.info/fetchmail-FAQ.html#R15\n"));
+                   }
 
                    /*
                     * If we're running in background, try to mail the
@@ -1106,49 +1191,53 @@ const int maxfetch;             /* maximum number of messages to fetch */
                     * we let the user know service is restored.
                     */
                    if (run.poll_interval
-                       && ctl->wehavesentauthnote
-                       && ((ctl->wehaveauthed && ++ctl->authfailcount == 10)
-                           || ++ctl->authfailcount == 3)
-                       && !open_warning_by_mail(ctl, (struct msgblk *)NULL))
+                       && !ctl->wehavesentauthnote
+                       && ((ctl->wehaveauthed && ++ctl->authfailcount >= 10)
+                           || (!ctl->wehaveauthed && ++ctl->authfailcount >= 3))
+                       && !open_warning_by_mail(ctl))
                    {
                        ctl->wehavesentauthnote = 1;
-                       stuff_warning(ctl,
-                                     GT_("Subject: fetchmail authentication failed on %s@%s\r\n"),
+                       stuff_warning(iana_charset, ctl,
+                                     GT_("Subject: fetchmail authentication failed on %s@%s"),
                            ctl->remotename, ctl->server.truename);
-                       stuff_warning(ctl,
-                                     GT_("Fetchmail could not get mail from %s@%s.\r\n"), 
+                       stuff_warning(NULL, ctl, "%s", "");
+                       stuff_warning(NULL, ctl,
+                                     GT_("Fetchmail could not get mail from %s@%s.\n"), 
                                      ctl->remotename,
                                      ctl->server.truename);
-                       if (ctl->wehaveauthed)
-                           stuff_warning(ctl, GT_("\
-The attempt to get authorization failed.\r\n\
-Since we have already succeeded in getting authorization for this\r\n\
-connection, this is probably another failure mode (such as busy server)\r\n\
-that fetchmail cannot distinguish because the server didn't send a useful\r\n\
-error message.\r\n\
-\r\n\
-However, if you HAVE changed you account details since starting the\r\n\
-fetchmail daemon, you need to stop the daemon, change your configuration\r\n\
-of fetchmail, and then restart the daemon.\r\n\
-\r\n\
-The fetchmail daemon will continue running and attempt to connect\r\n\
-at each cycle.  No future notifications will be sent until service\r\n\
+                       if (ctl->wehaveauthed) {
+                           stuff_warning(NULL, ctl, GT_("\
+The attempt to get authorization failed.\n\
+Since we have already succeeded in getting authorization for this\n\
+connection, this is probably another failure mode (such as busy server)\n\
+that fetchmail cannot distinguish because the server didn't send a useful\n\
+error message."));
+                           stuff_warning(NULL, ctl, GT_("\
+\n\
+However, if you HAVE changed your account details since starting the\n\
+fetchmail daemon, you need to stop the daemon, change your configuration\n\
+of fetchmail, and then restart the daemon.\n\
+\n\
+The fetchmail daemon will continue running and attempt to connect\n\
+at each cycle.  No future notifications will be sent until service\n\
 is restored."));
-                       else
-                           stuff_warning(ctl, GT_("\
-The attempt to get authorization failed.\r\n\
-This probably means your password is invalid, but some servers have\r\n\
-other failure modes that fetchmail cannot distinguish from this\r\n\
-because they don't send useful error messages on login failure.\r\n\
-\r\n\
-The fetchmail daemon will continue running and attempt to connect\r\n\
-at each cycle.  No future notifications will be sent until service\r\n\
+                       } else {
+                           stuff_warning(NULL, ctl, GT_("\
+The attempt to get authorization failed.\n\
+This probably means your password is invalid, but some servers have\n\
+other failure modes that fetchmail cannot distinguish from this\n\
+because they don't send useful error messages on login failure.\n\
+\n\
+The fetchmail daemon will continue running and attempt to connect\n\
+at each cycle.  No future notifications will be sent until service\n\
 is restored."));
+                       }
                        close_warning_by_mail(ctl, (struct msgblk *)NULL);
                    }
                }
                else if (err == PS_REPOLL)
                {
+                 if (outlevel >= O_VERBOSE)
                    report(stderr, GT_("Repoll immediately on %s@%s\n"),
                           ctl->remotename,
                           ctl->server.truename);
@@ -1181,17 +1270,18 @@ is restored."));
                           GT_("Authorization OK on %s@%s\n"),
                           ctl->remotename,
                           ctl->server.truename);
-                   if (!open_warning_by_mail(ctl, (struct msgblk *)NULL))
+                   if (!open_warning_by_mail(ctl))
                    {
-                       stuff_warning(ctl,
-                             GT_("Subject: fetchmail authentication OK on %s@%s\r\n"),
+                       stuff_warning(iana_charset, ctl,
+                             GT_("Subject: fetchmail authentication OK on %s@%s"), 
                                      ctl->remotename, ctl->server.truename);
-                       stuff_warning(ctl,
-                             GT_("Fetchmail was able to log into %s@%s.\r\n"), 
+                       stuff_warning(NULL, ctl, "%s", "");
+                       stuff_warning(NULL, ctl,
+                             GT_("Fetchmail was able to log into %s@%s.\n"), 
                                      ctl->remotename,
                                      ctl->server.truename);
-                       stuff_warning(ctl, 
-                                     GT_("Service has been restored.\r\n"));
+                       stuff_warning(NULL, ctl, 
+                                     GT_("Service has been restored.\n"));
                        close_warning_by_mail(ctl, (struct msgblk *)NULL);
                    
                    }
@@ -1211,9 +1301,11 @@ is restored."));
        /* now iterate over each folder selected */
        for (idp = ctl->mailboxes; idp; idp = idp->next)
        {
+           ctl->folder = idp->id;
            pass = 0;
            do {
                dispatches = 0;
+               transient_errors = 0;
                ++pass;
 
                /* reset timeout, in case we did an IDLE */
@@ -1229,42 +1321,35 @@ is restored."));
 
                /* compute # of messages and number of new messages waiting */
                stage = STAGE_GETRANGE;
-               err = (ctl->server.base_protocol->getrange)(mailserver_socket, ctl, idp->id, &count, &new, &bytes);
+               err = (ctl->server.base_protocol->getrange)(mailserver_socket, ctl, idp->id, &count, &newm, &bytes);
                if (err != 0)
                    goto cleanUp;
 
                /* show user how many messages we downloaded */
                if (idp->id)
-#ifdef HAVE_SNPRINTF
                    (void) snprintf(buf, sizeof(buf),
-#else
-                   (void) sprintf(buf,
-#endif /* HAVE_SNPRINTF */
                                   GT_("%s at %s (folder %s)"),
-                                  ctl->remotename, ctl->server.truename, idp->id);
+                                  ctl->remotename, ctl->server.pollname, idp->id);
                else
-#ifdef HAVE_SNPRINTF
-                   (void) snprintf(buf, sizeof(buf),
-#else
-                   (void) sprintf(buf,
-#endif /* HAVE_SNPRINTF */
-                              GT_("%s at %s"),
-                                  ctl->remotename, ctl->server.truename);
+                   (void) snprintf(buf, sizeof(buf), GT_("%s at %s"),
+                                  ctl->remotename, ctl->server.pollname);
                if (outlevel > O_SILENT)
                {
                    if (count == -1)            /* only used for ETRN */
                        report(stdout, GT_("Polling %s\n"), ctl->server.truename);
                    else if (count != 0)
                    {
-                       if (new != -1 && (count - new) > 0)
-                           report_build(stdout, GT_("%d %s (%d seen) for %s"),
-                                 count, count > 1 ? GT_("messages") :
-                                                    GT_("message"),
-                                 count-new, buf);
+                       if (newm != -1 && (count - newm) > 0)
+                           report_build(stdout, ngettext("%d message (%d %s) for %s", "%d messages (%d %s) for %s", (unsigned long)count),
+                                 count,
+                                 count - newm, 
+                                 ngettext("seen", "seen", (unsigned long)count-newm),
+                                 buf);
                        else
-                           report_build(stdout, GT_("%d %s for %s"), 
-                                 count, count > 1 ? GT_("messages") :
-                                                    GT_("message"), buf);
+                           report_build(stdout, ngettext("%d message for %s",
+                                                         "%d messages for %s",
+                                                         count), 
+                                 count, buf);
                        if (bytes == -1)
                            report_complete(stdout, ".\n");
                        else
@@ -1284,9 +1369,9 @@ is restored."));
 
                if (check_only)
                {
-                   if (new == -1 || ctl->fetchall)
-                       new = count;
-                   fetches = new     /* set error status correctly */
+                   if (newm == -1 || ctl->fetchall)
+                       newm = count;
+                   fetches = newm;     /* set error status correctly */
                    /*
                     * There used to be a `goto noerror' here, but this
                     * prevented checking of multiple folders.  This
@@ -1296,37 +1381,7 @@ is restored."));
                }
                else if (count > 0)
                {    
-                   flag        force_retrieval;
-                   int         i, num;
-
-                   /*
-                    * What forces this code is that in POP2 and
-                    * IMAP2bis you can't fetch a message without
-                    * having it marked `seen'.  In POP3 and IMAP4, on the
-                    * other hand, you can (peek_capable is set by 
-                    * each driver module to convey this; it's not a
-                    * method constant because of the difference between
-                    * IMAP2bis and IMAP4, and because POP3 doesn't  peek
-                    * if fetchall is on).
-                    *
-                    * The result of being unable to peek is that if there's
-                    * any kind of transient error (DNS lookup failure, or
-                    * sendmail refusing delivery due to process-table limits)
-                    * the message will be marked "seen" on the server without
-                    * having been delivered.  This is not a big problem if
-                    * fetchmail is running in foreground, because the user
-                    * will see a "skipped" message when it next runs and get
-                    * clued in.
-                    *
-                    * But in daemon mode this leads to the message
-                    * being silently ignored forever.  This is not
-                    * acceptable.
-                    *
-                    * We compensate for this by checking the error
-                    * count from the previous pass and forcing all
-                    * messages to be considered new if it's nonzero.
-                    */
-                   force_retrieval = !peek_capable && (ctl->errcount > 0);
+                   int         i;
 
                    /*
                     * Don't trust the message count passed by the server.
@@ -1335,27 +1390,31 @@ is restored."));
                     * count, and allocate a malloc area that would overlap
                     * a portion of the stack.
                     */
-                   if (count > INT_MAX/sizeof(int))
+                   if ((unsigned)count > INT_MAX/sizeof(int))
                    {
                        report(stderr, GT_("bogus message count!"));
-                       return(PS_PROTOCOL);
+                       err = PS_PROTOCOL;
+                       goto cleanUp;
                    }
 
-                   /* OK, we're going to gather size info next */
-                   xalloca(msgsizes, int *, sizeof(int) * count);
-                   xalloca(msgcodes, int *, sizeof(int) * count);
-                   for (i = 0; i < count; i++)
-                       msgcodes[i] = MSGLEN_UNKNOWN;
-
                    /* 
                     * We need the size of each message before it's
                     * loaded in order to pass it to the ESMTP SIZE
                     * option.  If the protocol has a getsizes method,
                     * we presume this means it doesn't get reliable
                     * sizes from message fetch responses.
+                    *
+                    * If the protocol supports getting sizes of subset of
+                    * messages, we skip this step now.
                     */
-                   if (proto->getsizes)
+                   if (proto->getsizes &&
+                       !(proto->getpartialsizes && NUM_NONZERO(ctl->fetchsizelimit)))
                    {
+                       xfree(msgsizes);
+                       msgsizes = (int *)xmalloc(sizeof(int) * count);
+                       for (i = 0; i < count; i++)
+                           msgsizes[i] = 0;
+
                        stage = STAGE_GETSIZES;
                        err = (proto->getsizes)(mailserver_socket, count, msgsizes);
                        if (err != 0)
@@ -1369,30 +1428,27 @@ is restored."));
                        }
                    }
 
-                   /* mark some messages not to be retrieved */
-                   for (num = 1; num <= count; num++)
-                   {
-                       if (NUM_NONZERO(ctl->limit) && (msgsizes[num-1] > ctl->limit))
-                           msgcodes[num-1] = MSGLEN_TOOLARGE;
-                       else if (ctl->fetchall || force_retrieval)
-                           continue;
-                       else if (ctl->server.base_protocol->is_old && (ctl->server.base_protocol->is_old)(mailserver_socket,ctl,num))
-                           msgcodes[num-1] = MSGLEN_OLD;
-/*                     else if (msgsizes[num-1] == 512)
-                               msgcodes[num-1] = MSGLEN_OLD;  (hmh) sample code to skip message */
-                   }
-
                    /* read, forward, and delete messages */
                    stage = STAGE_FETCH;
 
                    /* fetch in lockstep mode */
                    err = fetch_messages(mailserver_socket, ctl, 
-                                        count, msgsizes, msgcodes,
+                                        count, &msgsizes,
                                         maxfetch,
-                                        &fetches, &dispatches, &deletions);
-                   if (err)
+                                        &fetches, &dispatches, &deletions,
+                                        &transient_errors);
+                   if (err != PS_SUCCESS && err != PS_MAXFETCH)
                        goto cleanUp;
 
+                   if (transient_errors > MAX_TRANSIENT_ERRORS)
+                   {
+                       if (outlevel > O_SILENT)
+                       {
+                           report(stderr, GT_("Too many mails skipped (%d > %d) due to transient errors for %s\n"),
+                                   transient_errors, MAX_TRANSIENT_ERRORS, buf);
+                       }
+                   }
+
                    if (!check_only && ctl->skipped
                        && run.poll_interval > 0 && !nodetach)
                    {
@@ -1400,38 +1456,92 @@ is restored."));
                        send_size_warnings(ctl);
                    }
                }
+
+               /* end-of-mailbox processing before we repoll or switch to another one */
+               if (ctl->server.base_protocol->end_mailbox_poll)
+               {
+                   tmperr = (ctl->server.base_protocol->end_mailbox_poll)(mailserver_socket, ctl);
+                   if (tmperr) {
+                       err = tmperr;
+                       goto cleanUp;
+                   }
+               }
+               /* Return now if we have reached the fetchlimit */
+               if (maxfetch && maxfetch <= fetches)
+                   goto no_error;
            } while
                  /*
-                  * Only re-poll if we either had some actual forwards and 
-                  * either allowed deletions and had no errors.
+                  * Only repoll if we either had some actual forwards
+                  * or are idling for new mails and had no errors.
                   * Otherwise it is far too easy to get into infinite loops.
                   */
-                 (dispatches && ctl->server.base_protocol->retry && !ctl->keep && !ctl->errcount);
+                 (ctl->server.base_protocol->retry && (dispatches || ctl->idle) && !ctl->errcount);
        }
 
-    /* no_error: */
-       /* ordinary termination with no errors -- officially log out */
-       err = (ctl->server.base_protocol->logout_cmd)(mailserver_socket, ctl);
+       /* XXX: From this point onwards, preserve err unless a new error has occurred */
+
+    no_error:
+       /* PS_SUCCESS, PS_MAXFETCH: ordinary termination with no errors -- officially log out */
+       stage = STAGE_LOGOUT;
+       tmperr = (ctl->server.base_protocol->logout_cmd)(mailserver_socket, ctl);
+       if (tmperr != PS_SUCCESS)
+           err = tmperr;
        /*
         * Hmmmm...arguably this would be incorrect if we had fetches but
         * no dispatches (due to oversized messages, etc.)
         */
-       if (err == 0)
-           err = (fetches > 0) ? PS_SUCCESS : PS_NOMAIL;
+       else if (err == PS_SUCCESS && fetches == 0)
+           err = PS_NOMAIL;
+       /*
+        * Close all SMTP delivery sockets.  For optimum performance
+        * we'd like to hold them open til end of run, but (1) this
+        * loses if our poll interval is longer than the MTA's
+        * inactivity timeout, and (2) some MTAs (like smail) don't
+        * deliver after each message, but rather queue up mail and
+        * wait to actually deliver it until the input socket is
+        * closed.
+        *
+        * don't send QUIT for ODMR case because we're acting as a
+        * proxy between the SMTP server and client.
+        */
+       smtp_close(ctl, ctl->server.protocol != P_ODMR);
        cleanupSockClose(mailserver_socket);
        goto closeUp;
 
     cleanUp:
        /* we only get here on error */
-       if (err != 0 && err != PS_SOCKET)
+       if (err != 0 && err != PS_SOCKET && err != PS_REPOLL)
        {
            stage = STAGE_LOGOUT;
            (ctl->server.base_protocol->logout_cmd)(mailserver_socket, ctl);
        }
-       cleanupSockClose(mailserver_socket);
+
+       /* try to clean up all streams */
+       release_sink(ctl);
+       /*
+        * Sending SMTP QUIT on signal is theoretically nice, but led
+        * to a subtle bug.  If fetchmail was terminated by signal
+        * while it was shipping message text, it would hang forever
+        * waiting for a command acknowledge.  In theory we could
+        * enable the QUIT only outside of the message send.  In
+        * practice, we don't care.  All mailservers hang up on a
+        * dropped TCP/IP connection anyway.
+        */
+       smtp_close(ctl, 0);
+       if (mailserver_socket != -1) {
+           cleanupSockClose(mailserver_socket);
+       }
+       /* If there was a connect timeout, the socket should be closed.
+        * mailserver_socket_temp contains the socket to close.
+        */
+       if (mailserver_socket_temp != -1) {
+           cleanupSockClose(mailserver_socket_temp);
+           mailserver_socket_temp = -1;
+       }
     }
 
-    msg = (const char *)NULL;  /* sacrifice to -Wall */
+    /* no report on PS_AUTHFAIL */
+    msg = NULL;
     switch (err)
     {
     case PS_SOCKET:
@@ -1459,41 +1569,43 @@ is restored."));
        msg = GT_("DNS lookup");
        break;
     case PS_UNDEFINED:
-       report(stderr, GT_("undefined error\n"));
+       msg = GT_("undefined");
        break;
     }
-    /* no report on PS_MAXFETCH or PS_UNDEFINED or PS_AUTHFAIL */
-    if (err==PS_SOCKET || err==PS_SYNTAX
-               || err==PS_IOERR || err==PS_ERROR || err==PS_PROTOCOL 
-               || err==PS_LOCKBUSY || err==PS_SMTP || err==PS_DNS)
-    {
-       char    *stem;
-
-       if (phase == FORWARDING_WAIT || phase == LISTENER_WAIT)
-           stem = GT_("%s error while delivering to SMTP host %s\n");
+    if (msg) {
+       if (phase == FORWARDING_WAIT || phase == LISTENER_WAIT
+               || err == PS_SMTP)
+           report(stderr, GT_("%s error while fetching from %s@%s and delivering to SMTP host %s\n"),
+                   msg, ctl->remotename, ctl->server.pollname,
+                   ctl->smtphost ? ctl->smtphost : GT_("unknown"));
        else
-           stem = GT_("%s error while fetching from %s\n");
-       report(stderr, stem, msg, ctl->server.pollname);
+           report(stderr, GT_("%s error while fetching from %s@%s\n"),
+                   msg, ctl->remotename, ctl->server.pollname);
     }
 
 closeUp:
+    xfree(msgsizes);
+    ctl->folder = NULL;
+
     /* execute wrapup command, if any */
-    if (ctl->postconnect && (err = system(ctl->postconnect)))
+    if (ctl->postconnect && (tmperr = system(ctl->postconnect)))
     {
-       report(stderr, GT_("post-connection command failed with status %d\n"), err);
+       if (WIFSIGNALED(tmperr))
+           report(stderr, GT_("post-connection command terminated with signal %d\n"), WTERMSIG(tmperr));
+       else
+           report(stderr, GT_("post-connection command failed with status %d\n"), WEXITSTATUS(tmperr));
        if (err == PS_SUCCESS)
            err = PS_SYNTAX;
     }
 
-    signal(SIGALRM, alrmsave);
-    signal(SIGPIPE, pipesave);
+    set_timeout(0); /* cancel any pending alarm */
+    set_signal_handler(SIGALRM, alrmsave);
     return(err);
 }
 
-int do_protocol(ctl, proto)
-/* retrieve messages from server using given protocol method table */
-struct query *ctl;             /* parsed options with merged-in defaults */
-const struct method *proto;    /* protocol method table */
+/** retrieve messages from server using given protocol method table */
+int do_protocol(struct query *ctl /** parsed options with merged-in defaults */,
+               const struct method *proto /** protocol method table */)
 {
     int        err;
 
@@ -1530,7 +1642,8 @@ const struct method *proto;       /* protocol method table */
            return(PS_SYNTAX);
        }
     }
-    if (!proto->getsizes && NUM_SPECIFIED(ctl->limit))
+    if (!(proto->getsizes || proto->getpartialsizes)
+           && NUM_SPECIFIED(ctl->limit))
     {
        report(stderr,
                GT_("Option --limit is not supported with %s\n"),
@@ -1542,7 +1655,8 @@ const struct method *proto;       /* protocol method table */
      * If no expunge limit or we do expunges within the driver,
      * then just do one session, passing in any fetchlimit.
      */
-    if (proto->retry || !NUM_SPECIFIED(ctl->expunge))
+    if ((ctl->keep && !ctl->flush) ||
+       proto->retry || !NUM_SPECIFIED(ctl->expunge))
        return(do_session(ctl, proto, NUM_VALUE_OUT(ctl->fetchlimit)));
     /*
      * There's an expunge limit, and it isn't handled in the driver itself.