2 * \file uid.c -- UIDL handling for POP3 servers without LAST
4 * For license terms, see the file COPYING in this directory.
13 #if defined(STDC_HEADERS)
17 #if defined(HAVE_UNISTD_H)
21 #include "fetchmail.h"
26 * Machinery for handling UID lists live here. This is mainly to support
27 * RFC1725/RFC1939-conformant POP3 servers without a LAST command, but may also
28 * be useful for making the IMAP4 querying logic UID-oriented, if a future
29 * revision of IMAP forces me to.
31 * These functions are also used by the rest of the code to maintain
36 * At start of a query, we have a (possibly empty) list of UIDs to be
37 * considered seen in `oldsaved'. These are messages that were left in
38 * the mailbox and *not deleted* on previous queries (we don't need to
39 * remember the UIDs of deleted messages because ... well, they're gone!)
40 * This list is initially set up by initialize_saved_list() from the
43 * Early in the query, during the execution of the protocol-specific
44 * getrange code, the driver expects that the host's `newsaved' member
45 * will be filled with a list of UIDs and message numbers representing
46 * the mailbox state. If this list is empty, the server did
47 * not respond to the request for a UID listing.
49 * Each time a message is fetched, we can check its UID against the
50 * `oldsaved' list to see if it is old.
52 * Each time a message-id is seen, we mark it with MARK_SEEN.
54 * Each time a message is deleted, we mark its id UID_DELETED in the
55 * `newsaved' member. When we want to assert that an expunge has been
56 * done on the server, we call expunge_uid() to register that all
57 * deleted messages are gone by marking them UID_EXPUNGED.
59 * At the end of the query, the `newsaved' member becomes the
60 * `oldsaved' list. The old `oldsaved' list is freed.
62 * At the end of the fetchmail run, seen and non-EXPUNGED members of all
63 * current `oldsaved' lists are flushed out to the .fetchids file to
64 * be picked up by the next run. If there are no un-expunged
65 * messages, the file is deleted.
67 * One disadvantage of UIDL is that all the UIDs have to be downloaded
68 * before a search for new messages can be done. Typically, new messages
69 * are appended to mailboxes. Hence, downloading all UIDs just to download
70 * a few new mails is a waste of bandwidth. If new messages are always at
71 * the end of the mailbox, fast UIDL will decrease the time required to
74 * During fast UIDL, the UIDs of all messages are not downloaded! The first
75 * unseen message is searched for by using a binary search on UIDs. UIDs
76 * after the first unseen message are downloaded as and when needed.
78 * The advantages of fast UIDL are (this is noticeable only when the
79 * mailbox has too many mails):
81 * - There is no need to download the UIDs of all mails right at the start.
82 * - There is no need to save all the UIDs in memory separately in
84 * - There is no need to download the UIDs of seen mail (except for the
85 * first binary search).
86 * - The first new mail is downloaded considerably faster.
88 * The disadvantages are:
90 * - Since all UIDs are not downloaded, it is not possible to swap old and
91 * new list. The current state of the mailbox is essentially a merged state
92 * of old and new mails.
93 * - If an intermediate mail has been temporarily refused (say, due to 4xx
94 * code from the smtp server), this mail may not get downloaded.
95 * - If 'flush' is used, such intermediate mails will also get deleted.
97 * The first two disadvantages can be overcome by doing a linear search
98 * once in a while (say, every 10th poll). Also, with flush, fast UIDL
101 * Note: some comparisons (those used for DNS address lists) are caseblind!
106 /* UIDs associated with un-queried hosts */
107 static struct idlist *scratchlist;
110 void initialize_saved_lists(struct query *hostlist, const char *idfile)
111 /* read file of saved IDs and attach to each host */
117 /* make sure lists are initially empty */
118 for (ctl = hostlist; ctl; ctl = ctl->next) {
119 ctl->skipped = (struct idlist *)NULL;
120 ctl->oldsaved = (struct idlist *)NULL;
121 ctl->newsaved = (struct idlist *)NULL;
122 ctl->oldsavedend = &ctl->oldsaved;
128 * Croak if the uidl directory does not exist.
129 * This probably means an NFS mount failed and we can't
130 * see a uidl file that ought to be there.
131 * Question: is this a portable check? It's not clear
132 * that all implementations of lstat() will return ENOTDIR
133 * rather than plain ENOENT in this case...
135 if (lstat(idfile, &statbuf) < 0) {
136 if (errno == ENOTDIR)
138 report(stderr, "lstat: %s: %s\n", idfile, strerror(errno));
143 /* let's get stored message UIDs from previous queries */
144 if ((tmpfp = fopen(idfile, "r")) != (FILE *)NULL)
146 char buf[POPBUFSIZE+1];
147 char *host = NULL; /* pacify -Wall */
150 char *atsign; /* temp pointer used in parsing user and host */
154 char saveddelim2 = '\0'; /* pacify -Wall */
156 while (fgets(buf, POPBUFSIZE, tmpfp) != (char *)NULL)
159 * At this point, we assume the bug has two fields -- a user@host
160 * part, and an ID part. Either field may contain spurious @ signs.
161 * The previous version of this code presumed one could split at
162 * the rightmost '@'. This is not correct, as InterMail puts an
166 /* first, skip leading spaces */
167 user = buf + strspn(buf, " \t");
170 * First, we split the buf into a userhost part and an id
171 * part ... but id doesn't necessarily start with a '<',
172 * espescially if the POP server returns an X-UIDL header
173 * instead of a Message-ID, as GMX's (www.gmx.net) POP3
174 * StreamProxy V1.0 does.
176 * this is one other trick. The userhost part
177 * may contain ' ' in the user part, at least in
178 * the lotus notes case.
179 * So we start looking for the '@' after which the
180 * host will follow with the ' ' seperator finaly id.
182 * XXX FIXME: There is a case this code cannot handle:
183 * the user name cannot have blanks after a '@'.
185 if ((delimp1 = strchr(user, '@')) != NULL &&
186 (id = strchr(delimp1,' ')) != NULL)
188 for (delimp1 = id; delimp1 >= user; delimp1--)
189 if ((*delimp1 != ' ') && (*delimp1 != '\t'))
193 * It should be safe to assume that id starts after
194 * the " " - after all, we're writing the " "
195 * ourselves in write_saved_lists() :-)
197 id = id + strspn(id, " ");
199 delimp1++; /* but what if there is only white space ?!? */
200 /* we have at least one @, else we are not in this branch */
201 saveddelim1 = *delimp1; /* save char after token */
202 *delimp1 = '\0'; /* delimit token with \0 */
204 /* now remove trailing white space chars from id */
205 if ((delimp2 = strpbrk(id, " \t\n")) != NULL ) {
206 saveddelim2 = *delimp2;
210 atsign = strrchr(user, '@');
211 /* we have at least one @, else we are not in this branch */
215 /* find proper list and save it */
216 for (ctl = hostlist; ctl; ctl = ctl->next) {
217 if (strcasecmp(host, ctl->server.queryname) == 0
218 && strcasecmp(user, ctl->remotename) == 0) {
219 save_str(&ctl->oldsaved, id, UID_SEEN);
224 * If it's not in a host we're querying,
225 * save it anyway. Otherwise we'd lose UIDL
226 * information any time we queried an explicit
229 if (ctl == (struct query *)NULL) {
231 *delimp1 = saveddelim1;
233 if (delimp2 != NULL) {
234 *delimp2 = saveddelim2;
236 save_str(&scratchlist, buf, UID_SEEN);
240 fclose(tmpfp); /* not checking should be safe, mode was "r" */
243 if (outlevel >= O_DEBUG)
248 for (ctl = hostlist; ctl; ctl = ctl->next)
249 if (ctl->server.uidl)
251 report_build(stdout, GT_("Old UID list from %s:"),
252 ctl->server.pollname);
253 for (idp = ctl->oldsaved; idp; idp = idp->next)
254 report_build(stdout, " %s", idp->id);
256 report_build(stdout, GT_(" <empty>"));
257 report_complete(stdout, "\n");
263 report_build(stdout, GT_("Scratch list of UIDs:"));
264 for (idp = scratchlist; idp; idp = idp->next) {
265 char *t = sdump(idp->id, strlen(idp->id));
266 report_build(stdout, " %s", t);
270 report_build(stdout, GT_(" <empty>"));
271 report_complete(stdout, "\n");
275 #endif /* POP3_ENABLE */
277 /* return a pointer to the last element of the list to help the quick,
278 * constant-time addition to the list, NOTE: this function does not dup
279 * the string, the caller must do that. */
280 /*@shared@*/ static struct idlist **save_str_quick(/*@shared@*/ struct idlist **idl,
281 /*@only@*/ char *str, flag status)
282 /* save a number/UID pair on the given UID list */
286 /* do it nonrecursively so the list is in the right order */
287 for (end = idl; *end; end = &(*end)->next)
290 *end = (struct idlist *)xmalloc(sizeof(struct idlist));
292 (*end)->val.status.mark = status;
293 (*end)->val.status.num = 0;
299 /* return the end list element for direct modification */
300 struct idlist *save_str(struct idlist **idl, const char *str, flag st)
302 return *save_str_quick(idl, str ? xstrdup(str) : NULL, st);
305 void free_str_list(struct idlist **idl)
306 /* free the given UID list */
308 struct idlist *i = *idl;
311 struct idlist *t = i->next;
319 void save_str_pair(struct idlist **idl, const char *str1, const char *str2)
320 /* save an ID pair on the given list */
324 /* do it nonrecursively so the list is in the right order */
325 for (end = idl; *end; end = &(*end)->next)
328 *end = (struct idlist *)xmalloc(sizeof(struct idlist));
329 (*end)->id = str1 ? xstrdup(str1) : (char *)NULL;
331 (*end)->val.id2 = xstrdup(str2);
333 (*end)->val.id2 = (char *)NULL;
334 (*end)->next = (struct idlist *)NULL;
338 void free_str_pair_list(struct idlist **idl)
339 /* free the given ID pair list */
341 if (*idl == (struct idlist *)NULL)
344 free_idpair_list(&(*idl)->next);
346 free ((*idl)->val.id2);
348 *idl = (struct idlist *)NULL;
352 struct idlist *str_in_list(struct idlist **idl, const char *str, const flag caseblind)
353 /* is a given ID in the given list? (comparison may be caseblind) */
357 for( walk = *idl; walk; walk = walk->next )
358 if( strcasecmp( str, walk->id) == 0 )
361 for( walk = *idl; walk; walk = walk->next )
362 if( strcmp( str, walk->id) == 0 )
368 /** return the position of first occurrence of \a str in \a idl */
369 int str_nr_in_list(struct idlist **idl, const char *str)
376 for (walk = *idl, nr = 0; walk; nr ++, walk = walk->next)
377 if (strcmp(str, walk->id) == 0)
382 int str_nr_last_in_list( struct idlist **idl, const char *str)
383 /* return the last position of str in idl */
389 for( walk = *idl, nr = 0; walk; nr ++, walk = walk->next )
390 if( strcmp( str, walk->id) == 0 )
395 void str_set_mark( struct idlist **idl, const char *str, const flag val)
396 /* update the mark on an of an id to given value */
402 for(walk = *idl, nr = 0; walk; nr ++, walk = walk->next)
403 if (strcmp(str, walk->id) == 0)
404 walk->val.status.mark = val;
407 int count_list( struct idlist **idl)
408 /* count the number of elements in the list */
412 return 1 + count_list( &(*idl)->next );
415 /*@null@*/ char *str_from_nr_list(struct idlist **idl, long number)
416 /* return the number'th string in idl */
418 if( !*idl || number < 0)
422 return str_from_nr_list(&(*idl)->next, number-1);
426 char *str_find(struct idlist **idl, long number)
427 /* return the id of the given number in the given list. */
429 if (*idl == (struct idlist *) 0)
431 else if (number == (*idl)->val.status.num)
434 return(str_find(&(*idl)->next, number));
437 struct idlist *id_find(struct idlist **idl, long number)
438 /* return the id of the given number in the given list. */
441 for (idp = *idl; idp; idp = idp->next)
442 if (idp->val.status.num == number)
447 char *idpair_find(struct idlist **idl, const char *id)
448 /* return the id of the given id in the given list (caseblind comparison) */
450 if (*idl == (struct idlist *) 0)
452 else if (strcasecmp(id, (*idl)->id) == 0)
453 return((*idl)->val.id2 ? (*idl)->val.id2 : (*idl)->id);
455 return(idpair_find(&(*idl)->next, id));
458 int delete_str(struct idlist **idl, long num)
459 /* delete given message from given list */
463 for (idp = *idl; idp; idp = idp->next)
464 if (idp->val.status.num == num)
466 idp->val.status.mark = UID_DELETED;
472 struct idlist *copy_str_list(struct idlist *idl)
473 /* copy the given UID list */
475 struct idlist *newnode ;
477 if (idl == (struct idlist *)NULL)
481 newnode = (struct idlist *)xmalloc(sizeof(struct idlist));
482 memcpy(newnode, idl, sizeof(struct idlist));
483 newnode->next = copy_str_list(idl->next);
488 void append_str_list(struct idlist **idl, struct idlist **nidl)
489 /* append nidl to idl (does not copy *) */
491 if ((*nidl) == (struct idlist *)NULL || *nidl == *idl)
493 else if ((*idl) == (struct idlist *)NULL)
495 else if ((*idl)->next == (struct idlist *)NULL)
496 (*idl)->next = *nidl;
497 else if ((*idl)->next != *nidl)
498 append_str_list(&(*idl)->next, nidl);
502 void expunge_uids(struct query *ctl)
503 /* assert that all UIDs marked deleted have actually been expunged */
507 for (idl = dofastuidl ? ctl->oldsaved : ctl->newsaved; idl; idl = idl->next)
508 if (idl->val.status.mark == UID_DELETED)
509 idl->val.status.mark = UID_EXPUNGED;
512 void uid_swap_lists(struct query *ctl)
516 if (ctl->server.uidl && outlevel >= O_DEBUG)
521 report_build(stdout, GT_("Merged UID list from %s:"), ctl->server.pollname);
523 report_build(stdout, GT_("New UID list from %s:"), ctl->server.pollname);
524 for (idp = dofastuidl ? ctl->oldsaved : ctl->newsaved; idp; idp = idp->next) {
525 char *t = sdump(idp->id, strlen(idp->id));
526 report_build(stdout, " %s = %d", t, idp->val.status.mark);
530 report_build(stdout, GT_(" <empty>"));
531 report_complete(stdout, "\n");
535 * Don't swap UID lists unless we've actually seen UIDLs.
536 * This is necessary in order to keep UIDL information
537 * from being heedlessly deleted later on.
539 * Older versions of fetchmail did
541 * free_str_list(&scratchlist);
543 * after swap. This was wrong; we need to preserve the UIDL information
544 * from unqueried hosts. Unfortunately, not doing this means that
545 * under some circumstances UIDLs can end up being stored forever --
546 * specifically, if a user description is removed from .fetchmailrc
547 * with UIDLs from that account in .fetchids, there is no way for
548 * them to ever get garbage-collected.
552 /* old state of mailbox may now be irrelevant */
553 struct idlist *temp = ctl->oldsaved;
554 if (outlevel >= O_DEBUG)
555 report(stdout, GT_("swapping UID lists\n"));
556 ctl->oldsaved = ctl->newsaved;
557 ctl->newsaved = (struct idlist *) NULL;
558 free_str_list(&temp);
560 /* in fast uidl, there is no need to swap lists: the old state of
561 * mailbox cannot be discarded! */
562 else if (outlevel >= O_DEBUG && !dofastuidl)
563 report(stdout, GT_("not swapping UID lists, no UIDs seen this query\n"));
566 void uid_discard_new_list(struct query *ctl)
567 /* finish a query which had errors */
570 if (ctl->server.uidl && outlevel >= O_DEBUG)
574 /* this is now a merged list! the mails which were seen in this
575 * poll are marked here. */
576 report_build(stdout, GT_("Merged UID list from %s:"), ctl->server.pollname);
577 for (idp = ctl->oldsaved; idp; idp = idp->next) {
578 char *t = sdump(idp->id, strlen(idp->id));
579 report_build(stdout, " %s = %d", t, idp->val.status.mark);
583 report_build(stdout, GT_(" <empty>"));
584 report_complete(stdout, "\n");
589 /* new state of mailbox is not reliable */
590 if (outlevel >= O_DEBUG)
591 report(stdout, GT_("discarding new UID list\n"));
592 free_str_list(&ctl->newsaved);
593 ctl->newsaved = (struct idlist *) NULL;
597 void uid_reset_num(struct query *ctl)
598 /* reset the number associated with each id */
601 for (idp = ctl->oldsaved; idp; idp = idp->next)
602 idp->val.status.num = 0;
605 void write_saved_lists(struct query *hostlist, const char *idfile)
606 /* perform end-of-run write of seen-messages list */
613 /* if all lists are empty, nuke the file */
615 for (ctl = hostlist; ctl; ctl = ctl->next) {
616 for (idp = ctl->oldsaved; idp; idp = idp->next)
617 if (idp->val.status.mark == UID_SEEN
618 || idp->val.status.mark == UID_DELETED)
622 /* either nuke the file or write updated last-seen IDs */
623 if (!idcount && !scratchlist)
625 if (outlevel >= O_DEBUG) {
626 if (access(idfile, F_OK) == 0)
627 report(stdout, GT_("Deleting fetchids file.\n"));
629 if (unlink(idfile) && errno != ENOENT)
630 report(stderr, GT_("Error deleting %s: %s\n"), idfile, strerror(errno));
632 char *newnam = (char *)xmalloc(strlen(idfile) + 2);
633 strcpy(newnam, idfile);
635 if (outlevel >= O_DEBUG)
636 report(stdout, GT_("Writing fetchids file.\n"));
637 (void)unlink(newnam); /* remove file/link first */
638 if ((tmpfp = fopen(newnam, "w")) != (FILE *)NULL) {
640 for (ctl = hostlist; ctl; ctl = ctl->next) {
641 for (idp = ctl->oldsaved; idp; idp = idp->next)
642 if (idp->val.status.mark == UID_SEEN
643 || idp->val.status.mark == UID_DELETED)
644 fprintf(tmpfp, "%s@%s %s\n",
645 ctl->remotename, ctl->server.queryname, idp->id);
647 for (idp = scratchlist; idp; idp = idp->next)
648 fputs(idp->id, tmpfp);
650 errflg = ferror(tmpfp);
652 /* if we could write successfully, move into place;
655 report(stderr, GT_("Error writing to fetchids file %s, old file left in place.\n"), newnam);
658 if (rename(newnam, idfile)) {
659 report(stderr, GT_("Cannot rename fetchids file %s to %s: %s\n"), newnam, idfile, strerror(errno));
663 report(stderr, GT_("Cannot open fetchids file %s for writing: %s\n"), newnam, strerror(errno));
668 #endif /* POP3_ENABLE */
670 /* uid.c ends here */