2 * \file uid.c -- UIDL handling for POP3 servers without LAST
4 * For license terms, see the file COPYING in this directory.
13 #if defined(STDC_HEADERS)
17 #if defined(HAVE_UNISTD_H)
21 #include "fetchmail.h"
26 * Machinery for handling UID lists live here. This is currently used
27 * by POP3, but may also be useful for making the IMAP4 querying logic
30 * These functions are also used by the rest of the code to maintain
35 * At start of a query, we have a (possibly empty) list of UIDs to be
36 * considered seen in `oldsaved'. These are messages that were left in
37 * the mailbox and *not deleted* on previous queries (we don't need to
38 * remember the UIDs of deleted messages because ... well, they're gone!)
39 * This list is initially set up by initialize_saved_list() from the
42 * Early in the query, during the execution of the protocol-specific
43 * getrange code, the driver expects that the host's `newsaved' member
44 * will be filled with a list of UIDs and message numbers representing
45 * the mailbox state. If this list is empty, the server did
46 * not respond to the request for a UID listing.
48 * Each time a message is fetched, we can check its UID against the
49 * `oldsaved' list to see if it is old.
51 * Each time a message-id is seen, we mark it with MARK_SEEN.
53 * Each time a message is deleted, we mark its id UID_DELETED in the
54 * `newsaved' member. When we want to assert that an expunge has been
55 * done on the server, we call expunge_uid() to register that all
56 * deleted messages are gone by marking them UID_EXPUNGED.
58 * At the end of the query, the `newsaved' member becomes the
59 * `oldsaved' list. The old `oldsaved' list is freed.
61 * At the end of the fetchmail run, seen and non-EXPUNGED members of all
62 * current `oldsaved' lists are flushed out to the .fetchids file to
63 * be picked up by the next run. If there are no un-expunged
64 * messages, the file is deleted.
66 * One disadvantage of UIDL is that all the UIDs have to be downloaded
67 * before a search for new messages can be done. Typically, new messages
68 * are appended to mailboxes. Hence, downloading all UIDs just to download
69 * a few new mails is a waste of bandwidth. If new messages are always at
70 * the end of the mailbox, fast UIDL will decrease the time required to
73 * During fast UIDL, the UIDs of all messages are not downloaded! The first
74 * unseen message is searched for by using a binary search on UIDs. UIDs
75 * after the first unseen message are downloaded as and when needed.
77 * The advantages of fast UIDL are (this is noticeable only when the
78 * mailbox has too many mails):
80 * - There is no need to download the UIDs of all mails right at the start.
81 * - There is no need to save all the UIDs in memory separately in
83 * - There is no need to download the UIDs of seen mail (except for the
84 * first binary search).
85 * - The first new mail is downloaded considerably faster.
87 * The disadvantages are:
89 * - Since all UIDs are not downloaded, it is not possible to swap old and
90 * new list. The current state of the mailbox is essentially a merged state
91 * of old and new mails.
92 * - If an intermediate mail has been temporarily refused (say, due to 4xx
93 * code from the smtp server), this mail may not get downloaded.
94 * - If 'flush' is used, such intermediate mails will also get deleted.
96 * The first two disadvantages can be overcome by doing a linear search
97 * once in a while (say, every 10th poll). Also, with flush, fast UIDL
100 * Note: some comparisons (those used for DNS address lists) are caseblind!
106 /** UIDs associated with un-queried hosts */
107 static struct idlist *scratchlist;
109 /** Read saved IDs from \a idfile and attach to each host in \a hostlist. */
110 static int dump_saved_uid(struct uid_db_record *rec, void *unused)
116 t = sdump(rec->id, rec->id_len);
117 report_build(stdout, " %s\n", t);
123 /** Read saved IDs from \a idfile and attach to each host in \a hostlist.
124 * Returns 0 for success, or a non-zero error code. */
125 int initialize_saved_lists(struct query *hostlist, const char *idfile)
132 /* make sure lists are initially empty */
133 for (ctl = hostlist; ctl; ctl = ctl->next) {
134 ctl->skipped = (struct idlist *)NULL;
136 init_uid_db(&ctl->oldsaved);
137 init_uid_db(&ctl->newsaved);
143 * Croak if the uidl directory does not exist.
144 * This probably means an NFS mount failed and we can't
145 * see a uidl file that ought to be there.
146 * Question: is this a portable check? It's not clear
147 * that all implementations of lstat() will return ENOTDIR
148 * rather than plain ENOENT in this case...
150 if (lstat(idfile, &statbuf) < 0) {
151 if (errno == ENOTDIR)
153 report(stderr, "lstat: %s: %s\n", idfile, strerror(errno));
158 /* let's get stored message UIDs from previous queries */
159 if ((tmpfp = fopen(idfile, "r")) != (FILE *)NULL)
161 char buf[POPBUFSIZE+1];
162 char *host = NULL; /* pacify -Wall */
165 char *atsign; /* temp pointer used in parsing user and host */
169 char saveddelim2 = '\0'; /* pacify -Wall */
171 while (fgets(buf, POPBUFSIZE, tmpfp) != (char *)NULL)
174 * At this point, we assume the bug has two fields -- a user@host
175 * part, and an ID part. Either field may contain spurious @ signs.
176 * The previous version of this code presumed one could split at
177 * the rightmost '@'. This is not correct, as InterMail puts an
181 /* first, skip leading spaces */
182 user = buf + strspn(buf, " \t");
185 * First, we split the buf into a userhost part and an id
186 * part ... but id doesn't necessarily start with a '<',
187 * espescially if the POP server returns an X-UIDL header
188 * instead of a Message-ID, as GMX's (www.gmx.net) POP3
189 * StreamProxy V1.0 does.
191 * this is one other trick. The userhost part
192 * may contain ' ' in the user part, at least in
193 * the lotus notes case.
194 * So we start looking for the '@' after which the
195 * host will follow with the ' ' separator with the id.
197 * XXX FIXME: There is a case this code cannot handle:
198 * the user name cannot have blanks after a '@'.
200 if ((delimp1 = strchr(user, '@')) != NULL &&
201 (id = strchr(delimp1,' ')) != NULL)
203 for (delimp1 = id; delimp1 >= user; delimp1--)
204 if ((*delimp1 != ' ') && (*delimp1 != '\t'))
208 * It should be safe to assume that id starts after
209 * the " " - after all, we're writing the " "
210 * ourselves in write_saved_lists() :-)
212 id = id + strspn(id, " ");
214 delimp1++; /* but what if there is only white space ?!? */
215 /* we have at least one @, else we are not in this branch */
216 saveddelim1 = *delimp1; /* save char after token */
217 *delimp1 = '\0'; /* delimit token with \0 */
219 /* now remove trailing white space chars from id */
220 if ((delimp2 = strpbrk(id, " \t\n")) != NULL ) {
221 saveddelim2 = *delimp2;
225 atsign = strrchr(user, '@');
226 /* we have at least one @, else we are not in this branch */
230 /* find uidl db and save it */
231 for (ctl = hostlist; ctl; ctl = ctl->next) {
232 if (strcasecmp(host, ctl->server.queryname) == 0
233 && strcasecmp(user, ctl->remotename) == 0) {
234 uid_db_insert(&ctl->oldsaved, id, UID_SEEN);
239 * If it's not in a host we're querying,
240 * save it anyway. Otherwise we'd lose UIDL
241 * information any time we queried an explicit
244 if (ctl == (struct query *)NULL) {
246 *delimp1 = saveddelim1;
248 if (delimp2 != NULL) {
249 *delimp2 = saveddelim2;
251 save_str(&scratchlist, buf, UID_SEEN);
256 err |= fclose(tmpfp); /* not checking should be safe, mode was "r" */
257 /* bit-wise or, we only care about non-zero */
259 err = (errno != ENOENT);
262 report(stderr, GT_("Open or read error while reading idfile %s: %s\n"),
263 idfile, strerror(errno));
267 if (outlevel >= O_DEBUG)
271 for (ctl = hostlist; ctl; ctl = ctl->next)
273 report_build(stdout, GT_("Old UID list from %s:\n"),
274 ctl->server.pollname);
276 if (!uid_db_n_records(&ctl->oldsaved))
277 report_build(stdout, "%s\n", GT_(" <empty>"));
279 traverse_uid_db(&ctl->oldsaved, dump_saved_uid, NULL);
281 report_complete(stdout, "\n");
284 report_build(stdout, GT_("Scratch list of UIDs:\n"));
286 report_build(stdout, "%s\n", GT_(" <empty>"));
287 else for (idp = scratchlist; idp; idp = idp->next) {
288 char *t = sdump(idp->id, strlen(idp->id)-1);
289 report_build(stdout, " %s\n", t);
292 report_complete(stdout, "\n");
297 /** Assert that all UIDs marked deleted in query \a ctl have actually been
299 static int mark_as_expunged_if(struct uid_db_record *rec, void *unused)
303 if (rec->status == UID_DELETED) rec->status = UID_EXPUNGED;
307 void expunge_uids(struct query *ctl)
309 traverse_uid_db(dofastuidl ? &ctl->oldsaved : &ctl->newsaved,
310 mark_as_expunged_if, NULL);
313 static const char *str_uidmark(int mark)
327 if (snprintf(buf, sizeof(buf), "MARK=%d", mark) < 0)
334 static int dump_uid_db_record(struct uid_db_record *rec, void *arg)
339 n_recs = (unsigned int *)arg;
342 t = sdump(rec->id, rec->id_len);
343 report_build(stdout, " %s = %s\n", t, str_uidmark(rec->status));
349 static void dump_uid_db(struct uid_db *db)
353 n_recs = uid_db_n_records(db);
355 report_build(stdout, GT_(" <empty>"));
359 traverse_uid_db(db, dump_uid_db_record, &n_recs);
362 /** Finish a successful query */
363 void uid_swap_lists(struct query *ctl)
366 if (outlevel >= O_DEBUG)
369 report_build(stdout, GT_("Merged UID list from %s:\n"), ctl->server.pollname);
370 dump_uid_db(&ctl->oldsaved);
372 report_build(stdout, GT_("New UID list from %s:\n"), ctl->server.pollname);
373 dump_uid_db(&ctl->newsaved);
375 report_complete(stdout, "\n");
379 * Don't swap UID lists unless we've actually seen UIDLs.
380 * This is necessary in order to keep UIDL information
381 * from being heedlessly deleted later on.
383 * Older versions of fetchmail did
385 * free_str_list(&scratchlist);
387 * after swap. This was wrong; we need to preserve the UIDL information
388 * from unqueried hosts. Unfortunately, not doing this means that
389 * under some circumstances UIDLs can end up being stored forever --
390 * specifically, if a user description is removed from .fetchmailrc
391 * with UIDLs from that account in .fetchids, there is no way for
392 * them to ever get garbage-collected.
394 if (uid_db_n_records(&ctl->newsaved))
396 swap_uid_db_data(&ctl->newsaved, &ctl->oldsaved);
397 clear_uid_db(&ctl->newsaved);
399 /* in fast uidl, there is no need to swap lists: the old state of
400 * mailbox cannot be discarded! */
401 else if (outlevel >= O_DEBUG && !dofastuidl)
402 report(stdout, GT_("not swapping UID lists, no UIDs seen this query\n"));
405 /** Finish a query which had errors */
406 void uid_discard_new_list(struct query *ctl)
409 if (outlevel >= O_DEBUG)
411 /* this is now a merged list! the mails which were seen in this
412 * poll are marked here. */
413 report_build(stdout, GT_("Merged UID list from %s:\n"), ctl->server.pollname);
414 dump_uid_db(&ctl->oldsaved);
415 report_complete(stdout, "\n");
418 if (uid_db_n_records(&ctl->newsaved))
420 /* new state of mailbox is not reliable */
421 if (outlevel >= O_DEBUG)
422 report(stdout, GT_("discarding new UID list\n"));
423 clear_uid_db(&ctl->newsaved);
427 /** Reset the number associated with each id */
428 void uid_reset_num(struct query *ctl)
430 reset_uid_db_nums(&ctl->oldsaved);
433 /** Write list of seen messages, at end of run. */
434 static int count_seen_deleted(struct uid_db_record *rec, void *arg)
436 if (rec->status == UID_SEEN || rec->status == UID_DELETED)
441 struct write_saved_info {
446 static int write_uid_db_record(struct uid_db_record *rec, void *arg)
448 struct write_saved_info *info;
451 if (!(rec->status == UID_SEEN || rec->status == UID_DELETED))
454 info = (struct write_saved_info *)arg;
455 rc = fprintf(info->fp, "%s@%s %s\n",
456 info->ctl->remotename, info->ctl->server.queryname,
458 return rc < 0 ? -1 : 0;
461 /** Write new list of UIDs (state) to \a idfile. */
462 void write_saved_lists(struct query *hostlist, const char *idfile)
469 /* if all lists are empty, nuke the file */
471 for (ctl = hostlist; ctl; ctl = ctl->next)
472 traverse_uid_db(&ctl->oldsaved, count_seen_deleted, &idcount);
474 /* either nuke the file or write updated last-seen IDs */
475 if (!idcount && !scratchlist)
477 if (outlevel >= O_DEBUG) {
478 if (access(idfile, F_OK) == 0)
479 report(stdout, GT_("Deleting fetchids file.\n"));
481 if (unlink(idfile) && errno != ENOENT)
482 report(stderr, GT_("Error deleting %s: %s\n"), idfile, strerror(errno));
484 char *newnam = (char *)xmalloc(strlen(idfile) + 2);
486 strcpy(newnam, idfile);
488 if (outlevel >= O_DEBUG)
489 report(stdout, GT_("Writing fetchids file.\n"));
490 (void)unlink(newnam); /* remove file/link first */
491 old_umask = umask(S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH | S_IXOTH);
492 if ((tmpfp = fopen(newnam, "w")) != (FILE *)NULL) {
493 struct write_saved_info info;
498 for (ctl = hostlist; ctl; ctl = ctl->next) {
501 if (traverse_uid_db(&ctl->oldsaved, write_uid_db_record, &info) < 0) {
503 report(stderr, GT_("Write error on fetchids file %s: %s\n"), newnam, strerror(e));
509 for (idp = scratchlist; idp; idp = idp->next)
510 if (EOF == fputs(idp->id, tmpfp)) {
512 report(stderr, GT_("Write error on fetchids file %s: %s\n"), newnam, strerror(e));
518 (void)fflush(tmpfp); /* return code ignored, we check ferror instead */
519 errflg |= ferror(tmpfp);
520 errflg |= fclose(tmpfp);
521 /* if we could write successfully, move into place;
524 report(stderr, GT_("Error writing to fetchids file %s, old file left in place.\n"), newnam);
527 if (rename(newnam, idfile)) {
528 report(stderr, GT_("Cannot rename fetchids file %s to %s: %s\n"), newnam, idfile, strerror(errno));
532 report(stderr, GT_("Cannot open fetchids file %s for writing: %s\n"), newnam, strerror(errno));
535 (void)umask(old_umask);
538 #endif /* POP3_ENABLE */
540 /* uid.c ends here */