]> Pileus Git - ~andy/fetchmail/blobdiff - driver.c
Revise.
[~andy/fetchmail] / driver.c
index 380a7c2045067c0046de7b2a7ad04567e63148bc..905890d8a93e8fbf89b972ef456bb7d0a93cc343 100644 (file)
--- a/driver.c
+++ b/driver.c
 #if defined(HAVE_SYS_ITIMER_H)
 #include <sys/itimer.h>
 #endif
-#include  <sys/time.h>
 #include  <signal.h>
 #ifdef HAVE_SYS_WAIT_H
 #include <sys/wait.h>
 #endif
 
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
 #ifdef HAVE_NET_SOCKET_H
 #include <net/socket.h>
 #endif
-#ifdef HESIOD
+#ifdef HAVE_PKG_hesiod
 #include <hesiod.h>
 #endif
 
+#include <langinfo.h>
+
 #if defined(HAVE_RES_SEARCH) || defined(HAVE_GETHOSTBYNAME)
 #include <netdb.h>
 #include "mx.h"
@@ -67,12 +71,24 @@ int stage;          /* where are we? */
 int phase;             /* where are we, for error-logging purposes? */
 int batchcount;                /* count of messages sent in current batch */
 flag peek_capable;     /* can we peek for better error recovery? */
-int mailserver_socket_temp;    /* socket to free if connect timeout */ 
+int mailserver_socket_temp = -1;       /* socket to free if connect timeout */ 
 
-static int timeoutcount;               /* count consecutive timeouts */
+static volatile int timeoutcount = 0;  /* count consecutive timeouts */
+static volatile int idletimeout = 0;   /* timeout occured in idle stage? */
 
 static jmp_buf restart;
 
+int isidletimeout(void)
+/* last timeout occured in idle stage? */
+{
+    return idletimeout;
+}
+
+void resetidletimeout(void)
+{
+    idletimeout = 0;
+}
+
 void set_timeout(int timeleft)
 /* reset the nonresponse-timeout */
 {
@@ -89,34 +105,34 @@ void set_timeout(int timeleft)
 #endif
 }
 
-static void timeout_handler (int signal)
+static RETSIGTYPE timeout_handler (int signal)
 /* handle SIGALRM signal indicating a server timeout */
 {
-    timeoutcount++;
-    longjmp(restart, THROW_TIMEOUT);
+    if(stage != STAGE_IDLE) {
+       timeoutcount++;
+       longjmp(restart, THROW_TIMEOUT);
+    } else
+       idletimeout = 1;
 }
 
-static void sigpipe_handler (int signal)
+static RETSIGTYPE sigpipe_handler (int signal)
 /* handle SIGPIPE signal indicating a broken stream socket */
 {
     longjmp(restart, THROW_SIGPIPE);
 }
 
-/* ignore SIGALRM signal indicating a timeout during cleanup */
-static void cleanup_timeout_handler (int signal) { }
-
 #define CLEANUP_TIMEOUT 60 /* maximum timeout during cleanup */
 
 static int cleanupSockClose (int fd)
 /* close sockets in maximum CLEANUP_TIMEOUT seconds during cleanup */
 {
     int scerror;
-    void (*alrmsave)(int);
-    alrmsave = signal(SIGALRM, cleanup_timeout_handler);
+    SIGHANDLERTYPE alrmsave;
+    alrmsave = set_signal_handler(SIGALRM, null_signal_handler);
     set_timeout(CLEANUP_TIMEOUT);
     scerror = SockClose(fd);
     set_timeout(0);
-    signal(SIGALRM, alrmsave);
+    set_signal_handler(SIGALRM, alrmsave);
     return (scerror);
 }
 
@@ -198,7 +214,6 @@ const char *canonical;  /* server name */
     krb5_auth_context auth_context = NULL;
 
     krb5_init_context(&context);
-    krb5_init_ets(context);
     krb5_auth_con_init(context, &auth_context);
 
     if (retval = krb5_cc_default(context, &ccdef)) {
@@ -320,12 +335,15 @@ static void send_size_warnings(struct query *ctl)
      */
     if (open_warning_by_mail(ctl, (struct msgblk *)NULL))
        return;
-    stuff_warning(ctl,
-          GT_("Subject: Fetchmail oversized-messages warning.\r\n"
-            "\r\n"
-            "The following oversized messages remain on the mail server %s:"),
-                 ctl->server.pollname);
+    stuff_warning(iana_charset, ctl,
+          GT_("Subject: Fetchmail oversized-messages warning"));
+    stuff_warning(NULL, ctl, "");
+    stuff_warning(NULL, ctl,
+           GT_("The following oversized messages remain on the mail server %s:"),
+           ctl->server.pollname);
+
+    stuff_warning(NULL, ctl, "");
+
     if (run.poll_interval == 0)
        max_warning_poll_count = 0;
     else
@@ -338,8 +356,8 @@ static void send_size_warnings(struct query *ctl)
        {
            nbr = current->val.status.mark;
            size = atoi(current->id);
-           stuff_warning(ctl, 
-                   GT_("\t%d msg %d octets long skipped by fetchmail.\r\n"),
+           stuff_warning(NULL, ctl,
+                   GT_("  %d msg %d octets long skipped by fetchmail."),
                    nbr, size);
        }
        current->val.status.num++;
@@ -349,6 +367,8 @@ static void send_size_warnings(struct query *ctl)
            current->val.status.num = 0;
     }
 
+    stuff_warning(NULL, ctl, "");
+
     close_warning_by_mail(ctl, (struct msgblk *)NULL);
 }
 
@@ -360,12 +380,7 @@ static void mark_oversized(struct query *ctl, int num, int size)
     int cnt;
 
     /* convert size to string */
-#ifdef HAVE_SNPRINTF
-    snprintf(sizestr, sizeof(sizestr),
-#else
-    sprintf(sizestr,
-#endif /* HAVE_SNPRINTF */
-      "%d", size);
+    snprintf(sizestr, sizeof(sizestr), "%d", size);
 
     /* build a list of skipped messages
      * val.id = size of msg (string cnvt)
@@ -403,11 +418,58 @@ static void mark_oversized(struct query *ctl, int num, int size)
 }
 
 static int fetch_messages(int mailserver_socket, struct query *ctl, 
-                         int count, int *msgsizes, int *msgcodes, int maxfetch,
+                         int count, int *msgsizes, int maxfetch,
                          int *fetches, int *dispatches, int *deletions)
 /* fetch messages in lockstep mode */
 {
-    int num, err, len;
+    flag force_retrieval;
+    int num, firstnum = 1, lastnum = 0, err, len;
+    int fetchsizelimit = ctl->fetchsizelimit;
+    int msgsize;
+
+    if (ctl->server.base_protocol->getpartialsizes && NUM_NONZERO(fetchsizelimit))
+    {
+       /* for POP3, we can get the size of one mail only! Unfortunately, this
+        * protocol specific test cannot be done elsewhere as the protocol
+        * could be "auto". */
+       switch (ctl->server.protocol)
+       {
+           case P_POP3: case P_APOP: case P_RPOP:
+           fetchsizelimit = 1;
+       }
+
+       /* Time to allocate memory to store the sizes */
+       xalloca(msgsizes, int *, sizeof(int) * fetchsizelimit);
+    }
+
+    /*
+     * What forces this code is that in POP2 and
+     * IMAP2bis you can't fetch a message without
+     * having it marked `seen'.  In POP3 and IMAP4, on the
+     * other hand, you can (peek_capable is set by 
+     * each driver module to convey this; it's not a
+     * method constant because of the difference between
+     * IMAP2bis and IMAP4, and because POP3 doesn't  peek
+     * if fetchall is on).
+     *
+     * The result of being unable to peek is that if there's
+     * any kind of transient error (DNS lookup failure, or
+     * sendmail refusing delivery due to process-table limits)
+     * the message will be marked "seen" on the server without
+     * having been delivered.  This is not a big problem if
+     * fetchmail is running in foreground, because the user
+     * will see a "skipped" message when it next runs and get
+     * clued in.
+     *
+     * But in daemon mode this leads to the message
+     * being silently ignored forever.  This is not
+     * acceptable.
+     *
+     * We compensate for this by checking the error
+     * count from the previous pass and forcing all
+     * messages to be considered new if it's nonzero.
+     */
+    force_retrieval = !peek_capable && (ctl->errcount > 0);
 
     for (num = 1; num <= count; num++)
     {
@@ -415,18 +477,82 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
        flag suppress_forward = FALSE;
        flag suppress_readbody = FALSE;
        flag retained = FALSE;
+       int msgcode = MSGLEN_UNKNOWN;
 
-       if (msgcodes[num-1] < 0)
+       /* check if the message is old
+        * Note: the size of the message may not be known here */
+       if (ctl->fetchall || force_retrieval)
+           ;
+       else if (ctl->server.base_protocol->is_old && (ctl->server.base_protocol->is_old)(mailserver_socket,ctl,num))
+           msgcode = MSGLEN_OLD;
+       if (msgcode == MSGLEN_OLD)
        {
-           if ((msgcodes[num-1] == MSGLEN_TOOLARGE) && !check_only)
-               mark_oversized(ctl, num, msgsizes[num-1]);
+               /* To avoid flooding the syslog when using --keep,
+                * report "Skipped message" only when:
+                *  1) --verbose is on, or
+                *  2) fetchmail does not use syslog
+                */
+           if (   (outlevel >= O_VERBOSE) ||
+                  (outlevel > O_SILENT && !run.use_syslog)
+              )
+           {
+               report_build(stdout, 
+                            GT_("skipping message %s@%s:%d"),
+                            ctl->remotename, ctl->server.truename, num);
+           }
+
+           goto flagthemail;
+       }
+
+       if (ctl->server.base_protocol->getpartialsizes && NUM_NONZERO(fetchsizelimit) &&
+           lastnum < num)
+       {
+           /* Instead of getting the sizes of all mails at the start, we get
+            * the sizes in blocks of fetchsizelimit. This leads to better
+            * performance when there are too many mails (say, 10000) in
+            * the mailbox and either we are not getting all the mails at
+            * one go (--fetchlimit 100) or there is a frequent socket
+            * error while just getting the sizes of all mails! */
+
+           int i;
+           int oldstage = stage;
+           firstnum = num;
+           lastnum = num + fetchsizelimit - 1;
+           if (lastnum > count)
+               lastnum = count;
+           for (i = 0; i < fetchsizelimit; i++)
+               msgsizes[i] = 0;
+
+           stage = STAGE_GETSIZES;
+           err = (ctl->server.base_protocol->getpartialsizes)(mailserver_socket, num, lastnum, msgsizes);
+           if (err != 0)
+               return err;
+           stage = oldstage;
+       }
+
+       msgsize = msgsizes ? msgsizes[num-firstnum] : 0;
+
+       /* check if the message is oversized */
+       if (NUM_NONZERO(ctl->limit) && (msgsize > ctl->limit))
+           msgcode = MSGLEN_TOOLARGE;
+/*     else if (msgsize == 512)
+           msgcode = MSGLEN_OLD;  (hmh) sample code to skip message */
+
+       if (msgcode < 0)
+       {
+           if ((msgcode == MSGLEN_TOOLARGE) && !check_only)
+           {
+               mark_oversized(ctl, num, msgsize);
+               suppress_delete = TRUE;
+           }
            if (outlevel > O_SILENT)
            {
+               /* old messages are already handled above */
                report_build(stdout, 
                             GT_("skipping message %s@%s:%d (%d octets)"),
                             ctl->remotename, ctl->server.truename, num,
-                            msgsizes[num-1]);
-               switch (msgcodes[num-1])
+                            msgsize);
+               switch (msgcode)
                {
                case MSGLEN_INVALID:
                    /*
@@ -442,9 +568,7 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
                    report_build(stdout, GT_(" (length -1)"));
                    break;
                case MSGLEN_TOOLARGE:
-                   report_build(stdout, 
-                                GT_(" (oversized, %d octets)"),
-                                msgsizes[num-1]);
+                   report_build(stdout, GT_(" (oversized)"));
                    break;
                }
            }
@@ -452,15 +576,16 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
        else
        {
            flag wholesize = !ctl->server.base_protocol->fetch_body;
+           flag separatefetchbody = (ctl->server.base_protocol->fetch_body) ? TRUE : FALSE;
 
            /* request a message */
            err = (ctl->server.base_protocol->fetch_headers)(mailserver_socket,ctl,num, &len);
            if (err == PS_TRANSIENT)    /* server is probably Exchange */
            {
-               report_build(stdout,
-                            GT_("couldn't fetch headers, message %s@%s:%d (%d octets)"),
+               report(stdout,
+                            GT_("couldn't fetch headers, message %s@%s:%d (%d octets)\n"),
                             ctl->remotename, ctl->server.truename, num,
-                            msgsizes[num-1]);
+                            msgsize);
                continue;
            }
            else if (err != 0)
@@ -469,7 +594,7 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
            /* -1 means we didn't see a size in the response */
            if (len == -1)
            {
-               len = msgsizes[num - 1];
+               len = msgsize;
                wholesize = TRUE;
            }
 
@@ -480,8 +605,8 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
                             num, count);
 
                if (len > 0)
-                   report_build(stdout, GT_(" (%d %soctets)"),
-                                len, wholesize ? "" : GT_("header "));
+                   report_build(stdout, wholesize ? GT_(" (%d octets)")
+                                : GT_(" (%d header octets)"), len);
                if (outlevel >= O_VERBOSE)
                    report_complete(stdout, "\n");
                else
@@ -492,41 +617,24 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
             * Read the message headers and ship them to the
             * output sink.  
             */
-           err = readheaders(mailserver_socket, len, msgsizes[num-1],
-                            ctl, num);
+           err = readheaders(mailserver_socket, len, msgsize,
+                            ctl, num,
+                            /* pass the suppress_readbody flag only if the underlying
+                             * protocol does not fetch the body separately */
+                            separatefetchbody ? 0 : &suppress_readbody);
            if (err == PS_RETAINED)
-               suppress_readbody = suppress_forward = suppress_delete = retained = TRUE;
+               suppress_forward = suppress_delete = retained = TRUE;
            else if (err == PS_TRANSIENT)
                suppress_delete = suppress_forward = TRUE;
            else if (err == PS_REFUSED)
                suppress_forward = TRUE;
-#if 0
-           /* 
-            * readheaders does not read the body when it
-            * hits a non-header. It has been recently
-            * fixed to return PS_TRUNCATED (properly) when
-            * that happens, but apparently fixing that bug
-            * opened this one here (which looks like an 
-            * inproper fix from some ancient thinko)
-            */
            else if (err == PS_TRUNCATED)
                suppress_readbody = TRUE;
            else if (err)
                return(err);
-#else
-           else if (err && err != PS_TRUNCATED)
-               return(err);
-#endif
 
-           /* 
-            * If we're using IMAP4 or something else that
-            * can fetch headers separately from bodies,
-            * it's time to request the body now.  This
-            * fetch may be skipped if we got an anti-spam
-            * or other PS_REFUSED error response during
-            * readheaders.
-            */
-           if (ctl->server.base_protocol->fetch_body && !suppress_readbody) 
+           /* tell server we got it OK and resynchronize */
+           if (separatefetchbody && ctl->server.base_protocol->trail)
            {
                if (outlevel >= O_VERBOSE && !isafile(1))
                {
@@ -536,9 +644,27 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
 
                if ((err = (ctl->server.base_protocol->trail)(mailserver_socket, ctl, num)))
                    return(err);
-               len = 0;
-               if (!suppress_forward)
+           }
+
+           /* do not read the body which is not being forwarded only if
+            * the underlying protocol allows the body to be fetched
+            * separately */
+           if (separatefetchbody && suppress_forward)
+               suppress_readbody = TRUE;
+
+           /* 
+            * If we're using IMAP4 or something else that
+            * can fetch headers separately from bodies,
+            * it's time to request the body now.  This
+            * fetch may be skipped if we got an anti-spam
+            * or other PS_REFUSED error response during
+            * readheaders.
+            */
+           if (!suppress_readbody)
+           {
+               if (separatefetchbody)
                {
+                   len = -1;
                    if ((err=(ctl->server.base_protocol->fetch_body)(mailserver_socket,ctl,num,&len)))
                        return(err);
                    /*
@@ -549,27 +675,17 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
                     * string.  This violates RFC2060.
                     */
                    if (len == -1)
-                       len = msgsizes[num-1] - msgblk.msglen;
+                       len = msgsize - msgblk.msglen;
                    if (outlevel > O_SILENT && !wholesize)
                        report_complete(stdout,
                                        GT_(" (%d body octets) "), len);
                }
-           }
 
-           /* process the body now */
-           if (len > 0)
-           {
-               if (suppress_readbody)
-               {
-                   err = PS_SUCCESS;
-               }
-               else
-               {
-                   err = readbody(mailserver_socket,
-                                 ctl,
-                                 !suppress_forward,
-                                 len);
-               }
+               /* process the body now */
+               err = readbody(mailserver_socket,
+                             ctl,
+                             !suppress_forward,
+                             len);
                if (err == PS_TRANSIENT)
                    suppress_delete = suppress_forward = TRUE;
                else if (err)
@@ -616,13 +732,13 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
             * QUALCOMM server (at least) seems to be
             * reporting the on-disk size correctly.
             */
-           if (msgblk.msglen != msgsizes[num-1])
+           if (msgblk.msglen != msgsize)
            {
                if (outlevel >= O_DEBUG)
                    report(stdout,
                           GT_("message %s@%s:%d was not the expected length (%d actual != %d expected)\n"),
                           ctl->remotename, ctl->server.truename, num,
-                          msgblk.msglen, msgsizes[num-1]);
+                          msgblk.msglen, msgsize);
            }
 
            /* end-of-message processing starts here */
@@ -635,6 +751,7 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
                (*fetches)++;
        }
 
+flagthemail:
        /*
         * At this point in flow of control, either
         * we've bombed on a protocol error or had
@@ -646,25 +763,6 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
         * now.
         */
 
-       /*
-        * Tell the UID code we've seen this.
-        * Matthias Andree: only register the UID if we could actually
-        * forward this mail. If we omit this !suppress_delete check,
-        * fetchmail will never retry mail that the local listener
-        * refused temporarily.
-        */
-       if (ctl->newsaved && !suppress_delete)
-       {
-           struct idlist       *sdp;
-
-           for (sdp = ctl->newsaved; sdp; sdp = sdp->next)
-               if ((sdp->val.status.num == num) && (msgcodes[num-1] >= 0))
-               {
-                   sdp->val.status.mark = UID_SEEN;
-                   save_str(&ctl->oldsaved, sdp->id,UID_SEEN);
-               }
-       }
-
        /* maybe we delete this message now? */
        if (retained)
        {
@@ -673,7 +771,8 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
        }
        else if (ctl->server.base_protocol->delete
                 && !suppress_delete
-                && ((msgcodes[num-1] >= 0) ? !ctl->keep : ctl->flush))
+                && ((msgcode >= 0 && !ctl->keep)
+                    || (msgcode == MSGLEN_OLD && ctl->flush)))
        {
            (*deletions)++;
            if (outlevel > O_SILENT) 
@@ -681,17 +780,38 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
            err = (ctl->server.base_protocol->delete)(mailserver_socket, ctl, num);
            if (err != 0)
                return(err);
-#ifdef POP3_ENABLE
-           delete_str(&ctl->newsaved, num);
-#endif /* POP3_ENABLE */
        }
-       else if (outlevel > O_SILENT)
+       else
+       {
+           if (   (outlevel >= O_VERBOSE) ||
+                       /* To avoid flooding the syslog when using --keep,
+                        * report "Skipped message" only when:
+                        *  1) --verbose is on, or
+                        *  2) fetchmail does not use syslog, or
+                        *  3) the message was skipped for some other
+                        *     reason than just being old.
+                        */
+                  (outlevel > O_SILENT && (!run.use_syslog || msgcode != MSGLEN_OLD))
+              )
            report_complete(stdout, GT_(" not flushed\n"));
 
+           /* maybe we mark this message as seen now? */
+           if (ctl->server.base_protocol->mark_seen
+               && !suppress_delete
+               && (msgcode >= 0 && ctl->keep))
+           {
+               err = (ctl->server.base_protocol->mark_seen)(mailserver_socket, ctl, num);
+               if (err != 0)
+                   return(err);
+           }
+       }
+
        /* perhaps this as many as we're ready to handle */
-       if (maxfetch && maxfetch <= *fetches && *fetches < count)
+       if (maxfetch && maxfetch <= *fetches && num < count)
        {
-           report(stdout, GT_("fetchlimit %d reached; %d messages left on server %s account %s\n"),
+           report(stdout,
+                  ngettext("fetchlimit %d reached; %d message left on server %s account %s\n",
+                           "fetchlimit %d reached; %d messages left on server %s account %s\n", count - *fetches),
                   maxfetch, count - *fetches, ctl->server.truename, ctl->remotename);
            return(PS_MAXFETCH);
        }
@@ -700,37 +820,39 @@ static int fetch_messages(int mailserver_socket, struct query *ctl,
     return(PS_SUCCESS);
 }
 
-static int do_session(ctl, proto, maxfetch)
 /* retrieve messages from server using given protocol method table */
-struct query *ctl;             /* parsed options with merged-in defaults */
-const struct method *proto;    /* protocol method table */
-const int maxfetch;            /* maximum number of messages to fetch */
+static int do_session(
+       /* parsed options with merged-in defaults */
+       struct query *ctl,
+       /* protocol method table */
+       const struct method *proto,
+       /* maximum number of messages to fetch */
+       const int maxfetch)
 {
-    int js;
-#ifdef HAVE_VOLATILE
+    static int *msgsizes;
     volatile int err, mailserver_socket = -1;  /* pacifies -Wall */
-#else
-    int err, mailserver_socket = -1;
-#endif /* HAVE_VOLATILE */
+    int deletions = 0, js;
     const char *msg;
-    void (*pipesave)(int);
-    void (*alrmsave)(int);
+    SIGHANDLERTYPE pipesave;
+    SIGHANDLERTYPE alrmsave;
 
     ctl->server.base_protocol = proto;
 
+    msgsizes = NULL;
     pass = 0;
     err = 0;
     init_transact(proto);
 
     /* set up the server-nonresponse timeout */
-    alrmsave = signal(SIGALRM, timeout_handler);
+    alrmsave = set_signal_handler(SIGALRM, timeout_handler);
     mytimeout = ctl->server.timeout;
 
     /* set up the broken-pipe timeout */
-    pipesave = signal(SIGPIPE, sigpipe_handler);
+    pipesave = set_signal_handler(SIGPIPE, sigpipe_handler);
 
     if ((js = setjmp(restart)))
     {
+       /* exception caught */
 #ifdef HAVE_SIGPROCMASK
        /*
         * Don't rely on setjmp() to restore the blocked-signal mask.
@@ -747,19 +869,12 @@ const int maxfetch;               /* maximum number of messages to fetch */
        sigprocmask(SIG_UNBLOCK, &allsigs, NULL);
 #endif /* HAVE_SIGPROCMASK */
        
-       /* If there was a connect timeout, the socket should be closed.
-        * mailserver_socket_temp contains the socket to close.
-        */
-       mailserver_socket = mailserver_socket_temp;
-       
        if (js == THROW_SIGPIPE)
        {
-           signal(SIGPIPE, SIG_IGN);
+           set_signal_handler(SIGPIPE, SIG_IGN);
            report(stdout,
                   GT_("SIGPIPE thrown from an MDA or a stream socket error\n"));
            wait(0);
-           err = PS_SOCKET;
-           goto cleanUp;
        }
        else if (js == THROW_TIMEOUT)
        {
@@ -792,41 +907,33 @@ const int maxfetch;               /* maximum number of messages to fetch */
            if (timeoutcount > MAX_TIMEOUTS 
                && !open_warning_by_mail(ctl, (struct msgblk *)NULL))
            {
-               stuff_warning(ctl,
-                             GT_("Subject: fetchmail sees repeated timeouts\r\n"));
-               stuff_warning(ctl,
-                             GT_("Fetchmail saw more than %d timeouts while attempting to get mail from %s@%s.\r\n"), 
+               stuff_warning(iana_charset, ctl,
+                             GT_("Subject: fetchmail sees repeated timeouts"));
+               stuff_warning(NULL, ctl, "");
+               stuff_warning(NULL, ctl,
+                             GT_("Fetchmail saw more than %d timeouts while attempting to get mail from %s@%s.\n"), 
                              MAX_TIMEOUTS,
-                             ctl->remotename,
-                             ctl->server.truename);
-               stuff_warning(ctl, 
-    GT_("This could mean that your mailserver is stuck, or that your SMTP\r\n" \
-    "server is wedged, or that your mailbox file on the server has been\r\n" \
-    "corrupted by a server error.  You can run `fetchmail -v -v' to\r\n" \
-    "diagnose the problem.\r\n\r\n" \
-    "Fetchmail won't poll this mailbox again until you restart it.\r\n"));
+                             ctl->remotename, ctl->server.truename);
+               stuff_warning(NULL, ctl, 
+    GT_("This could mean that your mailserver is stuck, or that your SMTP\n" \
+    "server is wedged, or that your mailbox file on the server has been\n" \
+    "corrupted by a server error.  You can run `fetchmail -v -v' to\n" \
+    "diagnose the problem.\n\n" \
+    "Fetchmail won't poll this mailbox again until you restart it.\n"));
                close_warning_by_mail(ctl, (struct msgblk *)NULL);
                ctl->wedged = TRUE;
            }
-
-           err = PS_ERROR;
        }
 
-       /* try to clean up all streams */
-       release_sink(ctl);
-       smtp_close(ctl, 0);
-       if (mailserver_socket != -1) {
-           cleanupSockClose(mailserver_socket);
-           mailserver_socket = -1;
-       }
+       err = PS_SOCKET;
+       goto cleanUp;
     }
     else
     {
+       /* setjmp returned zero -> normal operation */
        char buf[MSGBUFSIZE+1], *realhost;
-       int count, new, bytes, deletions = 0;
-       int *msgsizes = (int *)NULL;
-       int *msgcodes = (int *)NULL;
-#if INET6_ENABLE
+       int count, new, bytes;
+#ifdef INET6_ENABLE
        int fetches, dispatches, oldphase;
 #else /* INET6_ENABLE */
        int port, fetches, dispatches, oldphase;
@@ -846,7 +953,7 @@ const int maxfetch;         /* maximum number of messages to fetch */
        oldphase = phase;
        phase = OPEN_WAIT;
        set_timeout(mytimeout);
-#if !INET6_ENABLE
+#ifndef INET6_ENABLE
 #ifdef SSL_ENABLE
        port = ctl->server.port ? ctl->server.port : ( ctl->use_ssl ? ctl->server.base_protocol->sslport : ctl->server.base_protocol->port );
 #else
@@ -854,7 +961,7 @@ const int maxfetch;         /* maximum number of messages to fetch */
 #endif
 #endif /* !INET6_ENABLE */
 
-#ifdef HESIOD
+#ifdef HAVE_PKG_hesiod
        /* If either the pollname or vianame are "hesiod" we want to
           lookup the user's hesiod pobox host */
        if (!strcasecmp(ctl->server.queryname, "hesiod")) {
@@ -901,8 +1008,35 @@ const int maxfetch;               /* maximum number of messages to fetch */
            }
            else
            {
+#ifdef INET6_ENABLE
+               struct addrinfo hints, *res;
+               int error;
+
+               memset(&hints, 0, sizeof(hints));
+               hints.ai_socktype = SOCK_STREAM;
+               hints.ai_family = AF_UNSPEC;
+               hints.ai_flags = AI_CANONNAME;
+
+               error = getaddrinfo(ctl->server.queryname, NULL, &hints, &res);
+               if (error)
+               {
+                   report(stderr,
+                          GT_("couldn't find canonical DNS name of %s (%s)\n"),
+                          ctl->server.pollname, ctl->server.queryname);
+                   err = PS_DNS;
+                   set_timeout(0);
+                   phase = oldphase;
+                   goto closeUp;
+               }
+               else
+               {
+                   ctl->server.truename=xstrdup(res->ai_canonname);
+                   ctl->server.trueaddr=xmalloc(res->ai_addrlen);
+                   memcpy(ctl->server.trueaddr, res->ai_addr, res->ai_addrlen);
+               }
+#else
                struct hostent  *namerec;
-                   
                /* 
                 * Get the host's IP, so we can report it like this:
                 *
@@ -913,8 +1047,8 @@ const int maxfetch;                /* maximum number of messages to fetch */
                if (namerec == (struct hostent *)NULL)
                {
                    report(stderr,
-                          GT_("couldn't find canonical DNS name of %s\n"),
-                          ctl->server.pollname);
+                          GT_("couldn't find canonical DNS name of %s (%s)\n"),
+                          ctl->server.pollname, ctl->server.queryname);
                    err = PS_DNS;
                    set_timeout(0);
                    phase = oldphase;
@@ -928,6 +1062,7 @@ const int maxfetch;                /* maximum number of messages to fetch */
                           namerec->h_addr_list[0],
                           namerec->h_length);
                }
+#endif
            }
        }
 #endif /* HAVE_GETHOSTBYNAME */
@@ -937,7 +1072,7 @@ const int maxfetch;                /* maximum number of messages to fetch */
        /* allow time for the port to be set up if we have a plugin */
        if (ctl->server.plugin)
            (void)sleep(1);
-#if INET6_ENABLE
+#ifdef INET6_ENABLE
        if ((mailserver_socket = SockOpen(realhost, 
                             ctl->server.service ? ctl->server.service : ( ctl->use_ssl ? ctl->server.base_protocol->sslservice : ctl->server.base_protocol->service ),
                             ctl->server.netsec, ctl->server.plugin)) == -1)
@@ -946,7 +1081,7 @@ const int maxfetch;                /* maximum number of messages to fetch */
 #endif /* INET6_ENABLE */
        {
            char        errbuf[BUFSIZ];
-#if !INET6_ENABLE
+#ifndef INET6_ENABLE
            int err_no = errno;
 #ifdef HAVE_RES_SEARCH
            if (err_no != 0 && h_errno != 0)
@@ -976,12 +1111,8 @@ const int maxfetch;               /* maximum number of messages to fetch */
                    else if (h_errno == TRY_AGAIN)
                        strcpy(errbuf, GT_("temporary name server error."));
                    else
-#ifdef HAVE_SNPRINTF
-                       snprintf(errbuf, sizeof(errbuf),
-#else
-                       sprintf(errbuf,
-#endif /* HAVE_SNPRINTF */
-                         GT_("unknown DNS error %d."), h_errno);
+                       snprintf (errbuf, sizeof(errbuf),
+                               GT_("unknown DNS error %d."), h_errno);
                }
                else
 #endif /* HAVE_RES_SEARCH */
@@ -998,12 +1129,12 @@ const int maxfetch;              /* maximum number of messages to fetch */
                /* warn the system administrator */
                if (open_warning_by_mail(ctl, (struct msgblk *)NULL) == 0)
                {
-                   stuff_warning(ctl,
-                        GT_("Subject: Fetchmail unreachable-server warning.\r\n"
-                          "\r\n"
-                          "Fetchmail could not reach the mail server %s:")
+                   stuff_warning(iana_charset, ctl,
+                        GT_("Subject: Fetchmail unreachable-server warning."));
+                   stuff_warning(NULL, ctl, "");
+                   stuff_warning(NULL, ctl, GT_("Fetchmail could not reach the mail server %s:"),
                                  ctl->server.pollname);
-                   stuff_warning(ctl, errbuf, ctl->server.pollname);
+                   stuff_warning(NULL, ctl, errbuf, ctl->server.pollname);
                    close_warning_by_mail(ctl, (struct msgblk *)NULL);
                }
 #endif
@@ -1016,8 +1147,8 @@ const int maxfetch;               /* maximum number of messages to fetch */
        }
 
 #ifdef SSL_ENABLE
-       /* Save the socket opened. Usefull if Fetchmail hangs on SSLOpen 
-        * because the socket can be closed
+       /* Save the socket opened. Useful if Fetchmail hangs on SSLOpen 
+        * because the socket can be closed.
         */
        mailserver_socket_temp = mailserver_socket;
        set_timeout(mytimeout);
@@ -1029,7 +1160,8 @@ const int maxfetch;               /* maximum number of messages to fetch */
            ctl->sslcertpath,ctl->sslfingerprint,realhost,ctl->server.pollname) == -1) 
        {
            report(stderr, GT_("SSL connection failed.\n"));
-           goto closeUp;
+           err = PS_AUTHFAIL;
+           goto cleanUp;
        }
        
        /* Fetchmail didn't hang on SSLOpen, 
@@ -1106,49 +1238,51 @@ const int maxfetch;             /* maximum number of messages to fetch */
                     * we let the user know service is restored.
                     */
                    if (run.poll_interval
-                       && ctl->wehavesentauthnote
-                       && ((ctl->wehaveauthed && ++ctl->authfailcount == 10)
-                           || ++ctl->authfailcount == 3)
+                       && !ctl->wehavesentauthnote
+                       && ((ctl->wehaveauthed && ++ctl->authfailcount >= 10)
+                           || (!ctl->wehaveauthed && ++ctl->authfailcount >= 3))
                        && !open_warning_by_mail(ctl, (struct msgblk *)NULL))
                    {
                        ctl->wehavesentauthnote = 1;
-                       stuff_warning(ctl,
-                                     GT_("Subject: fetchmail authentication failed on %s@%s\r\n"),
+                       stuff_warning(iana_charset, ctl,
+                                     GT_("Subject: fetchmail authentication failed on %s@%s"),
                            ctl->remotename, ctl->server.truename);
-                       stuff_warning(ctl,
-                                     GT_("Fetchmail could not get mail from %s@%s.\r\n"), 
+                       stuff_warning(NULL, ctl, "");
+                       stuff_warning(NULL, ctl,
+                                     GT_("Fetchmail could not get mail from %s@%s.\n"), 
                                      ctl->remotename,
                                      ctl->server.truename);
                        if (ctl->wehaveauthed)
-                           stuff_warning(ctl, GT_("\
-The attempt to get authorization failed.\r\n\
-Since we have already succeeded in getting authorization for this\r\n\
-connection, this is probably another failure mode (such as busy server)\r\n\
-that fetchmail cannot distinguish because the server didn't send a useful\r\n\
-error message.\r\n\
-\r\n\
-However, if you HAVE changed you account details since starting the\r\n\
-fetchmail daemon, you need to stop the daemon, change your configuration\r\n\
-of fetchmail, and then restart the daemon.\r\n\
-\r\n\
-The fetchmail daemon will continue running and attempt to connect\r\n\
-at each cycle.  No future notifications will be sent until service\r\n\
+                           stuff_warning(NULL, ctl, GT_("\
+The attempt to get authorization failed.\n\
+Since we have already succeeded in getting authorization for this\n\
+connection, this is probably another failure mode (such as busy server)\n\
+that fetchmail cannot distinguish because the server didn't send a useful\n\
+error message.\n\
+\n\
+However, if you HAVE changed your account details since starting the\n\
+fetchmail daemon, you need to stop the daemon, change your configuration\n\
+of fetchmail, and then restart the daemon.\n\
+\n\
+The fetchmail daemon will continue running and attempt to connect\n\
+at each cycle.  No future notifications will be sent until service\n\
 is restored."));
                        else
-                           stuff_warning(ctl, GT_("\
-The attempt to get authorization failed.\r\n\
-This probably means your password is invalid, but some servers have\r\n\
-other failure modes that fetchmail cannot distinguish from this\r\n\
-because they don't send useful error messages on login failure.\r\n\
-\r\n\
-The fetchmail daemon will continue running and attempt to connect\r\n\
-at each cycle.  No future notifications will be sent until service\r\n\
+                           stuff_warning(NULL, ctl, GT_("\
+The attempt to get authorization failed.\n\
+This probably means your password is invalid, but some servers have\n\
+other failure modes that fetchmail cannot distinguish from this\n\
+because they don't send useful error messages on login failure.\n\
+\n\
+The fetchmail daemon will continue running and attempt to connect\n\
+at each cycle.  No future notifications will be sent until service\n\
 is restored."));
                        close_warning_by_mail(ctl, (struct msgblk *)NULL);
                    }
                }
                else if (err == PS_REPOLL)
                {
+                 if (outlevel >= O_VERBOSE)
                    report(stderr, GT_("Repoll immediately on %s@%s\n"),
                           ctl->remotename,
                           ctl->server.truename);
@@ -1183,15 +1317,16 @@ is restored."));
                           ctl->server.truename);
                    if (!open_warning_by_mail(ctl, (struct msgblk *)NULL))
                    {
-                       stuff_warning(ctl,
-                             GT_("Subject: fetchmail authentication OK on %s@%s\r\n"),
+                       stuff_warning(iana_charset, ctl,
+                             GT_("Subject: fetchmail authentication OK on %s@%s"), 
                                      ctl->remotename, ctl->server.truename);
-                       stuff_warning(ctl,
-                             GT_("Fetchmail was able to log into %s@%s.\r\n"), 
+                       stuff_warning(NULL, ctl, "");
+                       stuff_warning(NULL, ctl,
+                             GT_("Fetchmail was able to log into %s@%s.\n"), 
                                      ctl->remotename,
                                      ctl->server.truename);
-                       stuff_warning(ctl, 
-                                     GT_("Service has been restored.\r\n"));
+                       stuff_warning(NULL, ctl, 
+                                     GT_("Service has been restored.\n"));
                        close_warning_by_mail(ctl, (struct msgblk *)NULL);
                    
                    }
@@ -1235,21 +1370,12 @@ is restored."));
 
                /* show user how many messages we downloaded */
                if (idp->id)
-#ifdef HAVE_SNPRINTF
                    (void) snprintf(buf, sizeof(buf),
-#else
-                   (void) sprintf(buf,
-#endif /* HAVE_SNPRINTF */
                                   GT_("%s at %s (folder %s)"),
-                                  ctl->remotename, ctl->server.truename, idp->id);
+                                  ctl->remotename, ctl->server.pollname, idp->id);
                else
-#ifdef HAVE_SNPRINTF
-                   (void) snprintf(buf, sizeof(buf),
-#else
-                   (void) sprintf(buf,
-#endif /* HAVE_SNPRINTF */
-                              GT_("%s at %s"),
-                                  ctl->remotename, ctl->server.truename);
+                   (void) snprintf(buf, sizeof(buf), GT_("%s at %s"),
+                                  ctl->remotename, ctl->server.pollname);
                if (outlevel > O_SILENT)
                {
                    if (count == -1)            /* only used for ETRN */
@@ -1257,14 +1383,16 @@ is restored."));
                    else if (count != 0)
                    {
                        if (new != -1 && (count - new) > 0)
-                           report_build(stdout, GT_("%d %s (%d seen) for %s"),
-                                 count, count > 1 ? GT_("messages") :
-                                                    GT_("message"),
-                                 count-new, buf);
+                           report_build(stdout, ngettext("%d message (%d %s) for %s", "%d messages (%d %s) for %s", (unsigned long)count),
+                                 count,
+                                 count-new, 
+                                 ngettext("seen", "seen", (unsigned long)count-new),
+                                 buf);
                        else
-                           report_build(stdout, GT_("%d %s for %s"), 
-                                 count, count > 1 ? GT_("messages") :
-                                                    GT_("message"), buf);
+                           report_build(stdout, ngettext("%d message for %s",
+                                                         "%d messages for %s",
+                                                         count), 
+                                 count, buf);
                        if (bytes == -1)
                            report_complete(stdout, ".\n");
                        else
@@ -1296,37 +1424,7 @@ is restored."));
                }
                else if (count > 0)
                {    
-                   flag        force_retrieval;
-                   int         i, num;
-
-                   /*
-                    * What forces this code is that in POP2 and
-                    * IMAP2bis you can't fetch a message without
-                    * having it marked `seen'.  In POP3 and IMAP4, on the
-                    * other hand, you can (peek_capable is set by 
-                    * each driver module to convey this; it's not a
-                    * method constant because of the difference between
-                    * IMAP2bis and IMAP4, and because POP3 doesn't  peek
-                    * if fetchall is on).
-                    *
-                    * The result of being unable to peek is that if there's
-                    * any kind of transient error (DNS lookup failure, or
-                    * sendmail refusing delivery due to process-table limits)
-                    * the message will be marked "seen" on the server without
-                    * having been delivered.  This is not a big problem if
-                    * fetchmail is running in foreground, because the user
-                    * will see a "skipped" message when it next runs and get
-                    * clued in.
-                    *
-                    * But in daemon mode this leads to the message
-                    * being silently ignored forever.  This is not
-                    * acceptable.
-                    *
-                    * We compensate for this by checking the error
-                    * count from the previous pass and forcing all
-                    * messages to be considered new if it's nonzero.
-                    */
-                   force_retrieval = !peek_capable && (ctl->errcount > 0);
+                   int         i;
 
                    /*
                     * Don't trust the message count passed by the server.
@@ -1341,21 +1439,23 @@ is restored."));
                        return(PS_PROTOCOL);
                    }
 
-                   /* OK, we're going to gather size info next */
-                   xalloca(msgsizes, int *, sizeof(int) * count);
-                   xalloca(msgcodes, int *, sizeof(int) * count);
-                   for (i = 0; i < count; i++)
-                       msgcodes[i] = MSGLEN_UNKNOWN;
-
                    /* 
                     * We need the size of each message before it's
                     * loaded in order to pass it to the ESMTP SIZE
                     * option.  If the protocol has a getsizes method,
                     * we presume this means it doesn't get reliable
                     * sizes from message fetch responses.
+                    *
+                    * If the protocol supports getting sizes of subset of
+                    * messages, we skip this step now.
                     */
-                   if (proto->getsizes)
+                   if (proto->getsizes &&
+                       !(proto->getpartialsizes && NUM_NONZERO(ctl->fetchsizelimit)))
                    {
+                       xalloca(msgsizes, int *, sizeof(int) * count);
+                       for (i = 0; i < count; i++)
+                           msgsizes[i] = 0;
+
                        stage = STAGE_GETSIZES;
                        err = (proto->getsizes)(mailserver_socket, count, msgsizes);
                        if (err != 0)
@@ -1369,25 +1469,12 @@ is restored."));
                        }
                    }
 
-                   /* mark some messages not to be retrieved */
-                   for (num = 1; num <= count; num++)
-                   {
-                       if (NUM_NONZERO(ctl->limit) && (msgsizes[num-1] > ctl->limit))
-                           msgcodes[num-1] = MSGLEN_TOOLARGE;
-                       else if (ctl->fetchall || force_retrieval)
-                           continue;
-                       else if (ctl->server.base_protocol->is_old && (ctl->server.base_protocol->is_old)(mailserver_socket,ctl,num))
-                           msgcodes[num-1] = MSGLEN_OLD;
-/*                     else if (msgsizes[num-1] == 512)
-                               msgcodes[num-1] = MSGLEN_OLD;  (hmh) sample code to skip message */
-                   }
-
                    /* read, forward, and delete messages */
                    stage = STAGE_FETCH;
 
                    /* fetch in lockstep mode */
                    err = fetch_messages(mailserver_socket, ctl, 
-                                        count, msgsizes, msgcodes,
+                                        count, msgsizes,
                                         maxfetch,
                                         &fetches, &dispatches, &deletions);
                    if (err)
@@ -1428,7 +1515,21 @@ is restored."));
            stage = STAGE_LOGOUT;
            (ctl->server.base_protocol->logout_cmd)(mailserver_socket, ctl);
        }
-       cleanupSockClose(mailserver_socket);
+
+       /* try to clean up all streams */
+       release_sink(ctl);
+       smtp_close(ctl, 0);
+       if (mailserver_socket != -1) {
+           cleanupSockClose(mailserver_socket);
+           mailserver_socket = -1;
+       }
+       /* If there was a connect timeout, the socket should be closed.
+        * mailserver_socket_temp contains the socket to close.
+        */
+       if (mailserver_socket_temp != -1) {
+           cleanupSockClose(mailserver_socket_temp);
+           mailserver_socket_temp = -1;
+       }
     }
 
     msg = (const char *)NULL;  /* sacrifice to -Wall */
@@ -1485,8 +1586,9 @@ closeUp:
            err = PS_SYNTAX;
     }
 
-    signal(SIGALRM, alrmsave);
-    signal(SIGPIPE, pipesave);
+    set_timeout(0); /* cancel any pending alarm */
+    set_signal_handler(SIGALRM, alrmsave);
+    set_signal_handler(SIGPIPE, pipesave);
     return(err);
 }
 
@@ -1542,7 +1644,8 @@ const struct method *proto;       /* protocol method table */
      * If no expunge limit or we do expunges within the driver,
      * then just do one session, passing in any fetchlimit.
      */
-    if (proto->retry || !NUM_SPECIFIED(ctl->expunge))
+    if ((ctl->keep && !ctl->flush) ||
+       proto->retry || !NUM_SPECIFIED(ctl->expunge))
        return(do_session(ctl, proto, NUM_VALUE_OUT(ctl->fetchlimit)));
     /*
      * There's an expunge limit, and it isn't handled in the driver itself.