2 * \file uid.c -- UIDL handling for POP3 servers without LAST
4 * For license terms, see the file COPYING in this directory.
17 #include "fetchmail.h"
22 * Machinery for handling UID lists live here. This is mainly to support
23 * RFC1725/RFC1939-conformant POP3 servers without a LAST command, but may also
24 * be useful for making the IMAP4 querying logic UID-oriented, if a future
25 * revision of IMAP forces me to.
27 * These functions are also used by the rest of the code to maintain
32 * At start of a query, we have a (possibly empty) list of UIDs to be
33 * considered seen in `oldsaved'. These are messages that were left in
34 * the mailbox and *not deleted* on previous queries (we don't need to
35 * remember the UIDs of deleted messages because ... well, they're gone!)
36 * This list is initially set up by initialize_saved_list() from the
39 * Early in the query, during the execution of the protocol-specific
40 * getrange code, the driver expects that the host's `newsaved' member
41 * will be filled with a list of UIDs and message numbers representing
42 * the mailbox state. If this list is empty, the server did
43 * not respond to the request for a UID listing.
45 * Each time a message is fetched, we can check its UID against the
46 * `oldsaved' list to see if it is old.
48 * Each time a message-id is seen, we mark it with MARK_SEEN.
50 * Each time a message is deleted, we mark its id UID_DELETED in the
51 * `newsaved' member. When we want to assert that an expunge has been
52 * done on the server, we call expunge_uid() to register that all
53 * deleted messages are gone by marking them UID_EXPUNGED.
55 * At the end of the query, the `newsaved' member becomes the
56 * `oldsaved' list. The old `oldsaved' list is freed.
58 * At the end of the fetchmail run, seen and non-EXPUNGED members of all
59 * current `oldsaved' lists are flushed out to the .fetchids file to
60 * be picked up by the next run. If there are no un-expunged
61 * messages, the file is deleted.
63 * One disadvantage of UIDL is that all the UIDs have to be downloaded
64 * before a search for new messages can be done. Typically, new messages
65 * are appended to mailboxes. Hence, downloading all UIDs just to download
66 * a few new mails is a waste of bandwidth. If new messages are always at
67 * the end of the mailbox, fast UIDL will decrease the time required to
70 * During fast UIDL, the UIDs of all messages are not downloaded! The first
71 * unseen message is searched for by using a binary search on UIDs. UIDs
72 * after the first unseen message are downloaded as and when needed.
74 * The advantages of fast UIDL are (this is noticeable only when the
75 * mailbox has too many mails):
77 * - There is no need to download the UIDs of all mails right at the start.
78 * - There is no need to save all the UIDs in memory separately in
80 * - There is no need to download the UIDs of seen mail (except for the
81 * first binary search).
82 * - The first new mail is downloaded considerably faster.
84 * The disadvantages are:
86 * - Since all UIDs are not downloaded, it is not possible to swap old and
87 * new list. The current state of the mailbox is essentially a merged state
88 * of old and new mails.
89 * - If an intermediate mail has been temporarily refused (say, due to 4xx
90 * code from the smtp server), this mail may not get downloaded.
91 * - If 'flush' is used, such intermediate mails will also get deleted.
93 * The first two disadvantages can be overcome by doing a linear search
94 * once in a while (say, every 10th poll). Also, with flush, fast UIDL
97 * Note: some comparisons (those used for DNS address lists) are caseblind!
103 /** UIDs associated with un-queried hosts */
104 static struct idlist *scratchlist;
106 /** Read saved IDs from \a idfile and attach to each host in \a hostlist. */
107 void initialize_saved_lists(struct query *hostlist, const char *idfile)
113 /* make sure lists are initially empty */
114 for (ctl = hostlist; ctl; ctl = ctl->next) {
115 ctl->skipped = (struct idlist *)NULL;
116 ctl->oldsaved = (struct idlist *)NULL;
117 ctl->newsaved = (struct idlist *)NULL;
118 ctl->oldsavedend = &ctl->oldsaved;
124 * Croak if the uidl directory does not exist.
125 * This probably means an NFS mount failed and we can't
126 * see a uidl file that ought to be there.
127 * Question: is this a portable check? It's not clear
128 * that all implementations of lstat() will return ENOTDIR
129 * rather than plain ENOENT in this case...
131 if (lstat(idfile, &statbuf) < 0) {
132 if (errno == ENOTDIR)
134 report(stderr, "lstat: %s: %s\n", idfile, strerror(errno));
139 /* let's get stored message UIDs from previous queries */
140 if ((tmpfp = fopen(idfile, "r")) != (FILE *)NULL)
142 char buf[POPBUFSIZE+1];
143 char *host = NULL; /* pacify -Wall */
146 char *atsign; /* temp pointer used in parsing user and host */
150 char saveddelim2 = '\0'; /* pacify -Wall */
152 while (fgets(buf, POPBUFSIZE, tmpfp) != (char *)NULL)
155 * At this point, we assume the bug has two fields -- a user@host
156 * part, and an ID part. Either field may contain spurious @ signs.
157 * The previous version of this code presumed one could split at
158 * the rightmost '@'. This is not correct, as InterMail puts an
162 /* first, skip leading spaces */
163 user = buf + strspn(buf, " \t");
166 * First, we split the buf into a userhost part and an id
167 * part ... but id doesn't necessarily start with a '<',
168 * espescially if the POP server returns an X-UIDL header
169 * instead of a Message-ID, as GMX's (www.gmx.net) POP3
170 * StreamProxy V1.0 does.
172 * this is one other trick. The userhost part
173 * may contain ' ' in the user part, at least in
174 * the lotus notes case.
175 * So we start looking for the '@' after which the
176 * host will follow with the ' ' separator with the id.
178 * XXX FIXME: There is a case this code cannot handle:
179 * the user name cannot have blanks after a '@'.
181 if ((delimp1 = strchr(user, '@')) != NULL &&
182 (id = strchr(delimp1,' ')) != NULL)
184 for (delimp1 = id; delimp1 >= user; delimp1--)
185 if ((*delimp1 != ' ') && (*delimp1 != '\t'))
189 * It should be safe to assume that id starts after
190 * the " " - after all, we're writing the " "
191 * ourselves in write_saved_lists() :-)
193 id = id + strspn(id, " ");
195 delimp1++; /* but what if there is only white space ?!? */
196 /* we have at least one @, else we are not in this branch */
197 saveddelim1 = *delimp1; /* save char after token */
198 *delimp1 = '\0'; /* delimit token with \0 */
200 /* now remove trailing white space chars from id */
201 if ((delimp2 = strpbrk(id, " \t\n")) != NULL ) {
202 saveddelim2 = *delimp2;
206 atsign = strrchr(user, '@');
207 /* we have at least one @, else we are not in this branch */
211 /* find proper list and save it */
212 for (ctl = hostlist; ctl; ctl = ctl->next) {
213 if (strcasecmp(host, ctl->server.queryname) == 0
214 && strcasecmp(user, ctl->remotename) == 0) {
215 save_str(&ctl->oldsaved, id, UID_SEEN);
220 * If it's not in a host we're querying,
221 * save it anyway. Otherwise we'd lose UIDL
222 * information any time we queried an explicit
225 if (ctl == (struct query *)NULL) {
227 *delimp1 = saveddelim1;
229 if (delimp2 != NULL) {
230 *delimp2 = saveddelim2;
232 save_str(&scratchlist, buf, UID_SEEN);
236 fclose(tmpfp); /* not checking should be safe, mode was "r" */
239 if (outlevel >= O_DEBUG)
243 for (ctl = hostlist; ctl; ctl = ctl->next)
245 report_build(stdout, GT_("Old UID list from %s:"),
246 ctl->server.pollname);
249 report_build(stdout, GT_(" <empty>"));
250 else for (idp = ctl->oldsaved; idp; idp = idp->next) {
251 char *t = sdump(idp->id, strlen(idp->id)-1);
252 report_build(stdout, " %s\n", t);
255 report_complete(stdout, "\n");
258 report_build(stdout, GT_("Scratch list of UIDs:"));
260 report_build(stdout, GT_(" <empty>"));
261 else for (idp = scratchlist; idp; idp = idp->next) {
262 char *t = sdump(idp->id, strlen(idp->id)-1);
263 report_build(stdout, " %s\n", t);
266 report_complete(stdout, "\n");
270 /** Assert that all UIDs marked deleted in query \a ctl have actually been
272 void expunge_uids(struct query *ctl)
276 for (idl = dofastuidl ? ctl->oldsaved : ctl->newsaved; idl; idl = idl->next)
277 if (idl->val.status.mark == UID_DELETED)
278 idl->val.status.mark = UID_EXPUNGED;
281 static const char *str_uidmark(int mark)
295 if (snprintf(buf, sizeof(buf), "MARK=%d", mark) < 0)
302 static void dump_list(const struct idlist *idp)
305 report_build(stdout, GT_(" <empty>"));
307 char *t = sdump(idp->id, strlen(idp->id));
308 report_build(stdout, " %s = %s%s", t, str_uidmark(idp->val.status.mark), idp->next ? "," : "");
315 void uid_swap_lists(struct query *ctl)
318 if (outlevel >= O_DEBUG)
321 report_build(stdout, GT_("Merged UID list from %s:"), ctl->server.pollname);
322 dump_list(ctl->oldsaved);
324 report_build(stdout, GT_("New UID list from %s:"), ctl->server.pollname);
325 dump_list(ctl->newsaved);
327 report_complete(stdout, "\n");
331 * Don't swap UID lists unless we've actually seen UIDLs.
332 * This is necessary in order to keep UIDL information
333 * from being heedlessly deleted later on.
335 * Older versions of fetchmail did
337 * free_str_list(&scratchlist);
339 * after swap. This was wrong; we need to preserve the UIDL information
340 * from unqueried hosts. Unfortunately, not doing this means that
341 * under some circumstances UIDLs can end up being stored forever --
342 * specifically, if a user description is removed from .fetchmailrc
343 * with UIDLs from that account in .fetchids, there is no way for
344 * them to ever get garbage-collected.
348 /* old state of mailbox may now be irrelevant */
349 struct idlist *temp = ctl->oldsaved;
350 if (outlevel >= O_DEBUG)
351 report(stdout, GT_("swapping UID lists\n"));
352 ctl->oldsaved = ctl->newsaved;
353 ctl->newsaved = (struct idlist *) NULL;
354 free_str_list(&temp);
356 /* in fast uidl, there is no need to swap lists: the old state of
357 * mailbox cannot be discarded! */
358 else if (outlevel >= O_DEBUG && !dofastuidl)
359 report(stdout, GT_("not swapping UID lists, no UIDs seen this query\n"));
362 /* finish a query which had errors */
363 void uid_discard_new_list(struct query *ctl)
366 if (outlevel >= O_DEBUG)
368 /* this is now a merged list! the mails which were seen in this
369 * poll are marked here. */
370 report_build(stdout, GT_("Merged UID list from %s:"), ctl->server.pollname);
371 dump_list(ctl->oldsaved);
372 report_complete(stdout, "\n");
377 /* new state of mailbox is not reliable */
378 if (outlevel >= O_DEBUG)
379 report(stdout, GT_("discarding new UID list\n"));
380 free_str_list(&ctl->newsaved);
381 ctl->newsaved = (struct idlist *) NULL;
385 /** Reset the number associated with each id */
386 void uid_reset_num(struct query *ctl)
389 for (idp = ctl->oldsaved; idp; idp = idp->next)
390 idp->val.status.num = 0;
393 /** Write list of seen messages, at end of run. */
394 void write_saved_lists(struct query *hostlist, const char *idfile)
401 /* if all lists are empty, nuke the file */
403 for (ctl = hostlist; ctl; ctl = ctl->next) {
404 for (idp = ctl->oldsaved; idp; idp = idp->next)
405 if (idp->val.status.mark == UID_SEEN
406 || idp->val.status.mark == UID_DELETED)
410 /* either nuke the file or write updated last-seen IDs */
411 if (!idcount && !scratchlist)
413 if (outlevel >= O_DEBUG) {
414 if (access(idfile, F_OK) == 0)
415 report(stdout, GT_("Deleting fetchids file.\n"));
417 if (unlink(idfile) && errno != ENOENT)
418 report(stderr, GT_("Error deleting %s: %s\n"), idfile, strerror(errno));
420 char *newnam = (char *)xmalloc(strlen(idfile) + 2);
421 strcpy(newnam, idfile);
423 if (outlevel >= O_DEBUG)
424 report(stdout, GT_("Writing fetchids file.\n"));
425 (void)unlink(newnam); /* remove file/link first */
426 if ((tmpfp = fopen(newnam, "w")) != (FILE *)NULL) {
428 for (ctl = hostlist; ctl; ctl = ctl->next) {
429 for (idp = ctl->oldsaved; idp; idp = idp->next)
430 if (idp->val.status.mark == UID_SEEN
431 || idp->val.status.mark == UID_DELETED)
432 if (fprintf(tmpfp, "%s@%s %s\n",
433 ctl->remotename, ctl->server.queryname, idp->id) < 0) {
435 report(stderr, GT_("Write error on fetchids file %s: %s\n"), newnam, strerror(e));
440 for (idp = scratchlist; idp; idp = idp->next)
441 if (EOF == fputs(idp->id, tmpfp)) {
443 report(stderr, GT_("Write error on fetchids file %s: %s\n"), newnam, strerror(e));
449 (void)fflush(tmpfp); /* return code ignored, we check ferror instead */
450 errflg |= ferror(tmpfp);
452 /* if we could write successfully, move into place;
455 report(stderr, GT_("Error writing to fetchids file %s, old file left in place.\n"), newnam);
458 if (rename(newnam, idfile)) {
459 report(stderr, GT_("Cannot rename fetchids file %s to %s: %s\n"), newnam, idfile, strerror(errno));
463 report(stderr, GT_("Cannot open fetchids file %s for writing: %s\n"), newnam, strerror(errno));
468 #endif /* POP3_ENABLE */
470 /* uid.c ends here */