From 50fd58d421d4b632d0980a5a22ed3506776f00ab Mon Sep 17 00:00:00 2001 From: Andy Spencer Date: Fri, 5 Feb 2010 13:03:25 +0000 Subject: [PATCH] Split gis-data into data and http parts gis-data: Generic things such as the callback types and cache types. gis-http: HTTP related caching functions (what was GisData) This was almost entirely refactored to be single threaded. It also uses a GisHttp structure to store data between calls to the fetch function. Plugins, etc should call the fetch function from a thread to avoid blocking the UI. --- src/data/Makefile.am | 2 + src/data/gis-data.c | 119 +------------------------------------------ src/data/gis-data.h | 36 +++++++------ src/data/gis-http.c | 116 +++++++++++++++++++++++++++++++++++++++++ src/data/gis-http.h | 50 ++++++++++++++++++ src/data/gis-wms.h | 9 ++-- src/gis.h | 1 + 7 files changed, 194 insertions(+), 139 deletions(-) create mode 100644 src/data/gis-http.c create mode 100644 src/data/gis-http.h diff --git a/src/data/Makefile.am b/src/data/Makefile.am index 15ebe47..6f2d78a 100644 --- a/src/data/Makefile.am +++ b/src/data/Makefile.am @@ -4,11 +4,13 @@ AM_CFLAGS += $(GLIB_CFLAGS) $(SOUP_CFLAGS) gis_data_includedir = $(includedir)/gis/data gis_data_include_HEADERS = \ gis-data.h \ + gis-http.h \ gis-wms.h noinst_LTLIBRARIES = libgis-data.la libgis_data_la_SOURCES = \ gis-data.c gis-data.h \ + gis-http.c gis-http.h \ gis-wms.c gis-wms.h libgis_data_la_LDFLAGS = -static diff --git a/src/data/gis-data.c b/src/data/gis-data.c index 82f267d..db3b16e 100644 --- a/src/data/gis-data.c +++ b/src/data/gis-data.c @@ -18,23 +18,13 @@ #include #include #include -#include #include "gis-data.h" -typedef struct { - gchar *uri; - gchar *local; - FILE *fp; - GisDataCacheDoneCallback user_done_cb; - GisDataCacheChunkCallback user_chunk_cb; - gpointer user_data; -} cache_file_end_t; - /* * Open a file, creating parent directories if needed */ -static FILE *fopen_p(const gchar *path, const gchar *mode) +FILE *fopen_p(const gchar *path, const gchar *mode) { gchar *parent = g_path_get_dirname(path); if (!g_file_test(parent, G_FILE_TEST_EXISTS)) @@ -42,110 +32,3 @@ static FILE *fopen_p(const gchar *path, const gchar *mode) g_free(parent); return fopen(path, mode); } - -static void done_cb(SoupSession *session, SoupMessage *message, gpointer _info) -{ - cache_file_end_t *info = _info; - g_debug("data: done_cb"); - - if (message->status_code == 416) - /* Range unsatisfiable, file already complete */ - info->user_done_cb(info->local, FALSE, info->user_data); - else if (SOUP_STATUS_IS_SUCCESSFUL(message->status_code)) - info->user_done_cb(info->local, TRUE, info->user_data); - else - g_warning("data: done_cb - error copying file, status=%d\n" - "\tsrc=%s\n" - "\tdst=%s", - message->status_code, info->uri, info->local); - g_free(info->uri); - g_free(info->local); - fclose(info->fp); - g_free(info); - //g_object_unref(session); This is probably leaking -} - -void chunk_cb(SoupMessage *message, SoupBuffer *chunk, gpointer _info) -{ - cache_file_end_t *info = _info; - if (!SOUP_STATUS_IS_SUCCESSFUL(message->status_code)) - return; - - if (!fwrite(chunk->data, chunk->length, 1, info->fp)) - g_error("data: chunk_cb - Unable to write data"); - goffset cur = ftell(info->fp); - //goffset total = soup_message_headers_get_range(message->response_headers); - goffset start=0, end=0, total=0; - soup_message_headers_get_content_range(message->response_headers, - &start, &end, &total); - - if (info->user_chunk_cb) - info->user_chunk_cb(info->local, cur, total, info->user_data); -} - -static SoupSession *do_cache(cache_file_end_t *info, gboolean truncate, gchar *reason) -{ - char *name = g_path_get_basename(info->uri); - g_debug("data: do_cache - Caching file %s: %s", name, reason); - g_free(name); - - /* TODO: move this to callback so we don't end up with 0 byte files - * Then change back to check for valid file after download */ - if (truncate) info->fp = fopen_p(info->local, "w"); - else info->fp = fopen_p(info->local, "a"); - long bytes = ftell(info->fp); - - SoupSession *session = soup_session_async_new(); - g_object_set(session, "user-agent", PACKAGE_STRING, NULL); - SoupMessage *message = soup_message_new("GET", info->uri); - if (message == NULL) - g_error("message is null, cannot parse uri"); - g_signal_connect(message, "got-chunk", G_CALLBACK(chunk_cb), info); - soup_message_headers_set_range(message->request_headers, bytes, -1); - soup_session_queue_message(session, message, done_cb, info); - return session; -} - -/* - * Cache a image from Ridge to the local disk - * \param path Path to the Ridge file, starting after /ridge/ - * \return The local path to the cached image - */ -SoupSession *cache_file(char *base, char *path, GisDataCacheType update, - GisDataCacheChunkCallback user_chunk_cb, - GisDataCacheDoneCallback user_done_cb, - gpointer user_data) -{ - g_debug("GisData: cache_file - base=%s, path=%s", base, path); - if (base == NULL) { - g_warning("GisData: cache_file - base is null"); - return NULL; - } - if (path == NULL) { - g_warning("GisData: cache_file - base is null"); - return NULL; - } - cache_file_end_t *info = g_malloc0(sizeof(cache_file_end_t)); - info->uri = g_strconcat(base, path, NULL); - info->local = g_build_filename(g_get_user_cache_dir(), PACKAGE, path, NULL); - info->fp = NULL; - info->user_chunk_cb = user_chunk_cb; - info->user_done_cb = user_done_cb; - info->user_data = user_data; - - if (update == GIS_REFRESH) - return do_cache(info, TRUE, "cache forced"); - - if (update == GIS_UPDATE) - return do_cache(info, FALSE, "attempting updating"); - - if (update == GIS_ONCE && !g_file_test(info->local, G_FILE_TEST_EXISTS)) - return do_cache(info, TRUE, "local does not exist"); - - /* No nead to cache, run the callback now and clean up */ - user_done_cb(info->local, FALSE, user_data); - g_free(info->uri); - g_free(info->local); - g_free(info); - return NULL; -} diff --git a/src/data/gis-data.h b/src/data/gis-data.h index a065650..7c8a728 100644 --- a/src/data/gis-data.h +++ b/src/data/gis-data.h @@ -15,27 +15,31 @@ * along with this program. If not, see . */ -#ifndef __DATA_H__ -#define __DATA_H__ +#ifndef __GIS_DATA_H__ +#define __GIS_DATA_H__ -#include +#include +/** + * Various ways to cach a file + */ typedef enum { - GIS_NEVER, // Never cache the file (for offline mode) - GIS_ONCE, // Cache the file if it does not exist - GIS_UPDATE, // Append additional data to cached copy (resume) - GIS_REFRESH, // Delete existing file and cache a new copy -} GisDataCacheType; - -typedef void (*GisDataCacheDoneCallback)(gchar *file, gboolean updated, - gpointer user_data); + GIS_LOCAL, // Only return local files (for offline mode) + GIS_ONCE, // Download the file only if it does not exist + GIS_UPDATE, // Update the file to be like the server + GIS_REFRESH, // Delete the existing file and fetch a new copy +} GisCacheType; -typedef void (*GisDataCacheChunkCallback)(gchar *file, goffset cur, +/** + * Function called when part of a file is fetched + * Used for updating progress bars, etc + */ +typedef void (*GisChunkCallback)(gchar *file, goffset cur, goffset total, gpointer user_data); -SoupSession *cache_file(char *base, char *path, GisDataCacheType update, - GisDataCacheChunkCallback user_chunk_cb, - GisDataCacheDoneCallback user_done_cb, - gpointer user_data); +/** + * Open a file and create the parent directory if necessasairy + */ +FILE *fopen_p(const gchar *path, const gchar *mode); #endif diff --git a/src/data/gis-http.c b/src/data/gis-http.c new file mode 100644 index 0000000..ad21baf --- /dev/null +++ b/src/data/gis-http.c @@ -0,0 +1,116 @@ +/* + * Copyright (C) 2009-2010 Andy Spencer + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include + +#include "gis-http.h" + +GisHttp *gis_http_new(const gchar *prefix) +{ + GisHttp *http = g_new0(GisHttp, 1); + http->prefix = g_strdup(prefix); + http->soup = soup_session_sync_new(); + g_object_set(http->soup, "user-agent", PACKAGE_STRING, NULL); + return http; +} + +/* For passing data to the chunck callback */ +struct _cache_info { + FILE *fp; + gchar *path; + GisChunkCallback callback; + gpointer user_data; +}; + +/** + * Append data to the file and call the users callback if they supplied one. + */ +static void _chunk_cb(SoupMessage *message, SoupBuffer *chunk, gpointer _info) +{ + struct _cache_info *info = _info; + + if (!SOUP_STATUS_IS_SUCCESSFUL(message->status_code)) { + g_warning("GisHttp: _chunk_cb - soup failed with %d", + message->status_code); + return; + } + + if (!fwrite(chunk->data, chunk->length, 1, info->fp)) + g_error("GisHttp: _chunk_cb - Unable to write data"); + + if (info->callback) { + goffset cur = ftell(info->fp); + goffset st=0, end=0, total=0; + soup_message_headers_get_content_range(message->response_headers, + &st, &end, &total); + info->callback(info->path, cur, total, info->user_data); + } +} + +/* TODO: use .part extentions and continue even when using GIS_ONCE */ +gchar *gis_http_fetch(GisHttp *self, const gchar *uri, const char *local, + GisCacheType mode, GisChunkCallback callback, gpointer user_data) +{ + g_debug("GisHttp: fetch - %s >> %s mode=%d", uri, self->prefix, mode); + + gchar *path = g_build_filename(g_get_user_cache_dir(), PACKAGE, + self->prefix, local, NULL); + + /* Unlink the file if we're refreshing it */ + if (mode == GIS_REFRESH) + g_remove(path); + + /* Do the cache if necessasairy */ + if (!(mode == GIS_ONCE && g_file_test(path, G_FILE_TEST_EXISTS)) && + mode != GIS_LOCAL) { + g_debug("GisHttp: do_cache - Caching file %s", local); + + /* Open the file for writting */ + FILE *fp = fopen_p(path, "a"); + + /* Make temp data */ + struct _cache_info info = { + .fp = fp, + .path = path, + .callback = callback, + .user_data = user_data, + }; + + /* Download the file */ + SoupMessage *message = soup_message_new("GET", uri); + if (message == NULL) + g_error("message is null, cannot parse uri"); + g_signal_connect(message, "got-chunk", G_CALLBACK(_chunk_cb), &info); + soup_message_headers_set_range(message->request_headers, ftell(fp), -1); + soup_session_send_message(self->soup, message); + + /* Finished */ + if (message->status_code == 416) { + /* Range unsatisfiable, file already complete */ + } else if (!SOUP_STATUS_IS_SUCCESSFUL(message->status_code)) + g_warning("GisHttp: done_cb - error copying file, status=%d\n" + "\tsrc=%s\n" + "\tdst=%s", + message->status_code, uri, path); + } + + /* TODO: free everything.. */ + return path; +} diff --git a/src/data/gis-http.h b/src/data/gis-http.h new file mode 100644 index 0000000..051df0b --- /dev/null +++ b/src/data/gis-http.h @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2009-2010 Andy Spencer + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef __GIS_HTTP_H__ +#define __GIS_HTTP_H__ + +#include +#include + +#include "gis-data.h" + +typedef struct _GisHttp { + gchar *prefix; + SoupSession *soup; +} GisHttp; + +/** + * @param prefix The cache prefix: e.g. /nexrad/level2/ + * @return The HTTP connection handle + */ +GisHttp *gis_http_new(const gchar *prefix); + +/** + * @param http GisHttp connection to use + * @param uri The uri to fetch + * @param mode Update type + * @param callback Callback to call when a chunck is recieved + * @param user_data Data to pass to callback + * @return The local path to the complete file + */ +gchar *gis_http_fetch(GisHttp *http, const gchar *uri, const gchar *local, + GisCacheType mode, + GisChunkCallback callback, + gpointer user_data); + +#endif diff --git a/src/data/gis-wms.h b/src/data/gis-wms.h index 2e35e65..07547e4 100644 --- a/src/data/gis-wms.h +++ b/src/data/gis-wms.h @@ -23,9 +23,7 @@ #include "objects/gis-tile.h" -typedef struct _GisWms GisWms; - -struct _GisWms { +typedef struct _GisWms { gchar *uri_prefix; gchar *uri_layer; gchar *uri_format; @@ -34,15 +32,16 @@ struct _GisWms { gint width; gint height; SoupSession *soup; -}; +} GisWms; -char *gis_wms_make_local(GisWms *wms, GisTile *tile); GisWms *gis_wms_new( gchar *uri_prefix, gchar *uri_layer, gchar *uri_format, gchar *cache_prefix, gchar *cache_ext, gint width, gint height); +char *gis_wms_make_local(GisWms *wms, GisTile *tile); + void gis_wms_free(GisWms *self); #endif diff --git a/src/gis.h b/src/gis.h index 9a4c2a7..6c91c03 100644 --- a/src/gis.h +++ b/src/gis.h @@ -26,6 +26,7 @@ /* GIS data */ #include +#include #include /* GIS objects */ -- 2.41.0