2 * Copyright (C) 2009-2011 Andy Spencer <andy753421@gmail.com>
4 * This program is free software: you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation, either version 3 of the License, or
7 * (at your option) any later version.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <http://www.gnu.org/licenses/>.
20 * @short_description: Hyper Text Transfer Protocol
22 * #GritsHttp is a small wrapper around libsoup to provide data access using
23 * the Hyper Text Transfer Protocol. Each #GritsHttp should be associated with
24 * a particular server or dataset, all the files downloaded for this dataset
25 * will be cached together in $HOME/.cache/grits/
30 #include <glib/gstdio.h>
31 #include <libsoup/soup.h>
33 #include "grits-http.h"
35 gchar *_get_cache_path(GritsHttp *http, const gchar *local)
37 return g_build_filename(g_get_user_cache_dir(), PACKAGE,
38 http->prefix, local, NULL);
43 * @prefix: The prefix in the cache to store the downloaded files.
44 * For example: * "/nexrad/level2/".
46 * Create a new #GritsHttp for the given prefix
48 * Returns: the new #GritsHttp
50 GritsHttp *grits_http_new(const gchar *prefix)
52 g_debug("GritsHttp: new - %s", prefix);
53 GritsHttp *http = g_new0(GritsHttp, 1);
54 http->soup = soup_session_sync_new();
55 http->prefix = g_strdup(prefix);
56 g_object_set(http->soup, "user-agent", PACKAGE_STRING, NULL);
57 g_object_set(http->soup, "timeout", 10, NULL);
63 * @http: the #GritsHttp to abort
65 * Cancels any pending requests and prevents new requests.
67 void grits_http_abort(GritsHttp *http)
69 g_debug("GritsHttp: abort - %s", http->prefix);
71 soup_session_abort(http->soup);
76 * @http: the #GritsHttp to free
78 * Frees resources used by @http and cancels any pending requests.
80 void grits_http_free(GritsHttp *http)
82 g_debug("GritsHttp: free - %s", http->prefix);
83 g_object_unref(http->soup);
88 /* For passing data to the chunck callback */
92 GritsChunkCallback callback;
95 struct _CacheInfoMain {
97 GritsChunkCallback callback;
102 /* call the user callback from the main thread,
103 * since it's usually UI updates */
104 static gboolean _chunk_main_cb(gpointer _infomain)
106 struct _CacheInfoMain *infomain = _infomain;
107 infomain->callback(infomain->path,
108 infomain->cur, infomain->total,
109 infomain->user_data);
115 * Append data to the file and call the users callback if they supplied one.
117 static void _chunk_cb(SoupMessage *message, SoupBuffer *chunk, gpointer _info)
119 struct _CacheInfo *info = _info;
121 if (!SOUP_STATUS_IS_SUCCESSFUL(message->status_code)) {
122 g_warning("GritsHttp: _chunk_cb - soup failed with %d",
123 message->status_code);
127 if (!fwrite(chunk->data, chunk->length, 1, info->fp))
128 g_error("GritsHttp: _chunk_cb - Unable to write data");
130 if (info->callback) {
131 struct _CacheInfoMain *infomain = g_new0(struct _CacheInfoMain, 1);
132 infomain->path = info->path;
133 infomain->callback = info->callback;
134 infomain->user_data = info->user_data;
135 infomain->cur = ftell(info->fp);
137 soup_message_headers_get_content_range(message->response_headers,
138 &st, &end, &infomain->total);
139 g_idle_add(_chunk_main_cb, infomain);
146 * @http: the #GritsHttp connection to use
147 * @uri: the URI to fetch
148 * @local: the local name to give to the file
149 * @mode: the update type to use when fetching data
150 * @callback: callback to call when a chunk of data is received
151 * @user_data: user data to pass to the callback
153 * Fetch a file from the cache. Whether the file is actually loaded from the
154 * remote server depends on the value of @mode.
156 * Returns: The local path to the complete file
158 /* TODO: use .part extentions and continue even when using GRITS_ONCE */
159 gchar *grits_http_fetch(GritsHttp *http, const gchar *uri, const char *local,
160 GritsCacheType mode, GritsChunkCallback callback, gpointer user_data)
162 g_debug("GritsHttp: fetch - %s mode=%d", local, mode);
164 g_debug("GritsPluginSat: _load_tile_thread - aborted");
167 gchar *path = _get_cache_path(http, local);
169 /* Unlink the file if we're refreshing it */
170 if (mode == GRITS_REFRESH)
173 /* Do the cache if necessasairy */
174 if (!(mode == GRITS_ONCE && g_file_test(path, G_FILE_TEST_EXISTS)) &&
175 mode != GRITS_LOCAL) {
176 g_debug("GritsHttp: fetch - Caching file %s", local);
178 /* Open the file for writting */
180 if (!g_file_test(path, G_FILE_TEST_EXISTS))
181 part = g_strdup_printf("%s.part", path);
182 FILE *fp = fopen_p(part, "ab");
184 g_warning("GritsHttp: fetch - error opening %s", path);
187 fseek(fp, 0, SEEK_END); // "a" is broken on Windows, twice
190 struct _CacheInfo info = {
193 .callback = callback,
194 .user_data = user_data,
197 /* Download the file */
198 SoupMessage *message = soup_message_new("GET", uri);
200 g_error("message is null, cannot parse uri");
201 g_signal_connect(message, "got-chunk", G_CALLBACK(_chunk_cb), &info);
203 soup_message_headers_set_range(message->request_headers, ftell(fp), -1);
204 if (mode == GRITS_REFRESH)
205 soup_message_headers_replace(message->request_headers,
206 "Cache-Control", "max-age=0");
207 soup_session_send_message(http->soup, message);
212 if (SOUP_STATUS_IS_SUCCESSFUL(message->status_code))
213 g_rename(part, path);
218 guint status = message->status_code;
219 g_object_unref(message);
220 if (status == SOUP_STATUS_CANCELLED) {
222 } else if (status == SOUP_STATUS_REQUESTED_RANGE_NOT_SATISFIABLE) {
223 /* Range unsatisfiable, file already complete */
224 } else if (!SOUP_STATUS_IS_SUCCESSFUL(status)) {
225 g_warning("GritsHttp: done_cb - error copying file, status=%d\n"
233 /* TODO: free everything.. */
238 * grits_http_available:
239 * @http: the #GritsHttp connection to use
240 * @filter: filter used to extract files from the index, or NULL
241 * For example: "href=\"([^"]*)\""
242 * @cache: path to the local cache, or NULL to not search the cache
243 * @extract: regex used to extract filenames from the page, should match the
244 * filename as $1, or NULL to use /http="([^"])"/
245 * @index: path to the index page, or NULL to not search online
247 * Look through the cache and an HTTP index page for a list of available files.
248 * The name of each file that matches the filter is added to the returned list.
250 * The list as well as the strings contained in it should be freed afterwards.
252 * Returns the list of matching filenames
254 GList *grits_http_available(GritsHttp *http,
255 gchar *filter, gchar *cache,
256 gchar *extract, gchar *index)
258 g_debug("GritsHttp: available - %s~=%s %s~=%s",
259 filter, cache, extract, index);
260 GRegex *filter_re = g_regex_new(filter, 0, 0, NULL);
263 /* Add cached files */
266 gchar *path = _get_cache_path(http, cache);
267 GDir *dir = g_dir_open(path, 0, NULL);
268 while (dir && (file = g_dir_read_name(dir)))
269 if (g_regex_match(filter_re, file, 0, NULL))
270 files = g_list_prepend(files, g_strdup(file));
276 /* Add online files if online */
279 g_snprintf(tmp, sizeof(tmp), ".index.%x", g_random_int());
280 gchar *path = grits_http_fetch(http, index, tmp,
281 GRITS_REFRESH, NULL, NULL);
285 g_file_get_contents(path, &html, NULL, NULL);
289 /* Match hrefs by default, this regex is not very accurate */
290 GRegex *extract_re = g_regex_new(
291 extract ?: "href=\"([^\"]*)\"", 0, 0, NULL);
293 g_regex_match(extract_re, html, 0, &info);
294 while (g_match_info_matches(info)) {
295 gchar *file = g_match_info_fetch(info, 1);
297 if (g_regex_match(filter_re, file, 0, NULL))
298 files = g_list_prepend(files, file);
302 g_match_info_next(info, NULL);
305 g_regex_unref(extract_re);
306 g_match_info_free(info);
312 g_regex_unref(filter_re);