From 19650fce02758614577e2df6e9838416d687f43d Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Tue, 19 Jan 2016 11:00:59 -0700 Subject: [PATCH 01/35] started working on S3 storage backend --- src/store_s3.c | 243 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 243 insertions(+) create mode 100644 src/store_s3.c diff --git a/src/store_s3.c b/src/store_s3.c new file mode 100644 index 00000000..d1441784 --- /dev/null +++ b/src/store_s3.c @@ -0,0 +1,243 @@ +#include "config.h" +#include +#include +#include +#include +#include +#include +#include + +#define HAVE_LIBS3 1 + +#ifdef HAVE_LIBS3 +#include +#endif + +#include "store.h" +#include "store_s3.h" +#include "render_config.h" +#include "protocol.h" + +#ifdef HAVE_LIBS3 + +static pthread_mutex_t qLock; +static int store_s3_initialized = 0; + +struct tile_cache +{ + struct stat_info st_stat; + char *tile; + int x, y, z; + char xmlname[XMLCONFIG_MAX]; +}; + +struct store_s3_ctx +{ + S3BucketContext* ctx; + const char *basepath; + struct tile_cache cache; +}; + +struct MemoryStruct +{ + char *memory; + size_t size; +}; + +static char* store_s3_xyz_to_storagekey(struct storage_backend *store, int x, int y, int z, char *key) +{ + snprintf(key, PATH_MAX - 1, "/%s/%i/%i/%i.png", ((struct store_s3_ctx *) (store->storage_ctx))->basepath, z, x, y); + return key; +} + +static int store_s3_tile_retrieve(struct storage_backend * store, const char *xmlconfig, const char *options, int x, int y, int z) +{ + struct store_s3_ctx * ctx = (struct store_s3_ctx *) (store->storage_ctx); + char * path; + + if ((ctx->cache.x == x) && (ctx->cache.y == y) && (ctx->cache.z == z) + && (strcmp(ctx->cache.xmlname, xmlconfig) == 0)) { + log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_fetch: Got a cached tile"); + return 1; + } + + log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_fetch: Fetching tile"); + + path = malloc(PATH_MAX); + + store_s3_xyz_to_storagekey(store, x, y, z, path); + log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_fetch: retrieving file %s", path); + + S3Status res = S3_get_object(ctx->ctx, path, NULL, 0, 0, NULL, handler, NULL); + free(path); + + if (res != S3StatusOK) { + log_message(STORE_LOGLVL_ERR, "store_s3_tile_fetch: failed to retrieve file: %s", S3_get_status_name(res)); + ctx->cache.x = -1; + ctx->cache.y = -1; + ctx->cache.z = -1; + return -1; + } + + if (ctx->cache.tile != NULL) + free(ctx->cache.tile); + ctx->cache.tile = chunk.memory; + ctx->cache.st_stat.size = chunk.size; + ctx->cache.st_stat.expired = 0; + res = + curl_easy_getinfo(ctx->ctx, CURLINFO_FILETIME, &(ctx->cache.st_stat.mtime)); + ctx->cache.st_stat.atime = 0; + log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_read: Read file of size %i", chunk.size); + break; + + ctx->cache.x = x; + ctx->cache.y = y; + ctx->cache.z = z; + strcpy(ctx->cache.xmlname, xmlconfig); + return 1; +} + +static int store_s3_tile_read(struct storage_backend * store, const char *xmlconfig, const char *options, int x, int y, int z, char *buf, size_t sz, int * compressed, char * log_msg) +{ + struct store_s3_ctx * ctx = (struct store_s3_ctx *) (store->storage_ctx); + + if (store_s3_tile_retrieve(store, xmlconfig, options, x, y, z) > 0) { + if (ctx->cache.st_stat.size > sz) { + log_message(STORE_LOGLVL_ERR, "store_s3_tile_read: size was too big, overrun %i %i", sz, ctx->cache.st_stat.size); + return -1; + } + memcpy(buf, ctx->cache.tile, ctx->cache.st_stat.size); + return ctx->cache.st_stat.size; + } else { + log_message(STORE_LOGLVL_ERR, "store_s3_tile_read: Fetching didn't work"); + return -1; + } +} + +static struct stat_info store_s3_tile_stat(struct storage_backend * store, const char *xmlconfig, const char *options, int x, int y, int z) +{ + struct stat_info tile_stat; + struct store_s3_ctx * ctx = (struct store_s3_ctx *) (store->storage_ctx); + + if (store_s3_tile_retrieve(store, xmlconfig, options, x, y, z) > 0) { + return ctx->cache.st_stat; + } else { + tile_stat.size = -1; + tile_stat.expired = 0; + tile_stat.mtime = 0; + tile_stat.atime = 0; + tile_stat.ctime = 0; + return tile_stat; + } +} + +static char * store_s3_tile_storage_id(struct storage_backend * store, const char *xmlconfig, const char *options, int x, int y, int z, char * string) +{ + + return store_s3_xyz_to_storagekey(store, x, y, z, string); +} + +static int store_s3_metatile_write(struct storage_backend * store, const char *xmlconfig, const char *options, int x, int y, int z, const char *buf, int sz) +{ + log_message(STORE_LOGLVL_ERR, "store_s3_metatile_write: This is a readonly storage backend. Write functionality isn't implemented"); + return -1; +} + +static int store_s3_metatile_delete(struct storage_backend * store, const char *xmlconfig, int x, int y, int z) +{ + log_message(STORE_LOGLVL_ERR, "store_s3_metatile_expire: This is a readonly storage backend. Write functionality isn't implemented"); + return -1; +} + +static int store_s3_metatile_expire(struct storage_backend *store, const char *xmlconfig, int x, int y, int z) +{ + + log_message(STORE_LOGLVL_ERR, "store_s3_metatile_expire: This is a readonly storage backend. Write functionality isn't implemented"); + return -1; +} + +static int store_s3_close_storage(struct storage_backend *store) +{ + struct store_s3_ctx * ctx = (struct store_s3_ctx *) (store->storage_ctx); + + free(ctx->baseurl); + if (ctx->cache.tile) + free(ctx->cache.tile); + S3_deinitialize(); + free(ctx); + free(store); + + return 0; +} + +#endif //Have curl + +struct storage_backend * init_storage_s3(const char *connection_string) +{ +#ifndef HAVE_LIBS3 + log_message(STORE_LOGLVL_ERR, + "init_storage_s3: Support for libs3 and therefore S3 storage has not been compiled into this program"); + return NULL; +#else + struct storage_backend *store = malloc(sizeof(struct storage_backend)); + struct store_s3_ctx *ctx = malloc(sizeof(struct store_s3_ctx)); + + S3Status res; + + log_message(STORE_LOGLVL_DEBUG, "init_storage_s3: initializing S3 storage backend for %s", connection_string); + + if (!store || !ctx) { + log_message(STORE_LOGLVL_ERR, "init_storage_s3: failed to allocate memory for context"); + if (store) + free(store); + if (ctx) + free(ctx); + return NULL; + } + + ctx->cache.x = -1; + ctx->cache.y = -1; + ctx->cache.z = -1; + ctx->cache.tile = NULL; + ctx->cache.xmlname[0] = 0; + + pthread_mutex_lock(&qLock); + if (!store_s3_initialized) { + log_message(STORE_LOGLVL_DEBUG, "init_storage_s3: Global init of curl", connection_string); + res = S3_initialize(NULL, S3_INIT_ALL, NULL); + store_s3_initialized = 1; + } else { + res = S3StatusOK; + } + pthread_mutex_unlock(&qLock); + if (res != S3StatusOK) { + log_message(STORE_LOGLVL_ERR, "init_storage_s3: failed to initialize S3 library: %s", S3_get_status_name(res)); + free(ctx); + free(store); + return NULL; + } + + // parse out the context information from the URL: s3://:@/[/] + ctx->ctx = malloc(sizeof(struct S3BucketContext)); + + char *fullurl = strdup(connection_string); + fullurl = &fullurl[5]; // advance past "s3://" + ctx->ctx->accessKeyId = strsep(&fullurl, ":"); + ctx->ctx->secretAccessKey = strsep(&fullurl, "@"); + ctx->ctx->hostName = strsep(&fullurl, "/"); + ctx->ctx->bucketName = strsep(&fullurl, "/"); + ctx->basepath = fullurl; + + store->storage_ctx = ctx; + + store->tile_read = &store_s3_tile_read; + store->tile_stat = &store_s3_tile_stat; + store->metatile_write = &store_s3_metatile_write; + store->metatile_delete = &store_s3_metatile_delete; + store->metatile_expire = &store_s3_metatile_expire; + store->tile_storage_id = &store_s3_tile_storage_id; + store->close_storage = &store_s3_close_storage; + + return store; +#endif +} From 0d8061333b0b679746e88dbcc30c320f7cfab76f Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Tue, 19 Jan 2016 11:02:15 -0700 Subject: [PATCH 02/35] added libs3 autoconf support, completed initial store_s3 implementation --- Makefile.am | 4 +- configure.ac | 5 + includes/store_s3.h | 16 +++ src/store_s3.c | 306 +++++++++++++++++++++++++++++++++++--------- 4 files changed, 271 insertions(+), 60 deletions(-) create mode 100644 includes/store_s3.h diff --git a/Makefile.am b/Makefile.am index 7fe98fac..360a0f58 100644 --- a/Makefile.am +++ b/Makefile.am @@ -7,8 +7,8 @@ ACLOCAL_AMFLAGS = -I m4 AM_CPPFLAGS = $(PTHREAD_CFLAGS) -DSYSTEM_LIBINIPARSER=@SYSTEM_LIBINIPARSER@ -STORE_SOURCES = src/store.c src/store_file.c src/store_file_utils.c src/store_memcached.c src/store_rados.c src/store_ro_http_proxy.c src/store_ro_composite.c src/store_null.c -STORE_LDFLAGS = $(LIBMEMCACHED_LDFLAGS) $(LIBRADOS_LDFLAGS) $(LIBCURL) +STORE_SOURCES = src/store.c src/store_file.c src/store_file_utils.c src/store_memcached.c src/store_rados.c src/store_ro_http_proxy.c src/store_ro_composite.c src/store_null.c src/store_s3.c +STORE_LDFLAGS = $(LIBMEMCACHED_LDFLAGS) $(LIBRADOS_LDFLAGS) $(LIBCURL) $(LIBS3_LDFLAGS) STORE_CPPFLAGS = bin_PROGRAMS = renderd render_expired render_list render_speedtest render_old diff --git a/configure.ac b/configure.ac index c6ff10ad..1965f377 100644 --- a/configure.ac +++ b/configure.ac @@ -53,6 +53,11 @@ AC_CHECK_LIB(rados, rados_version, [ LIBRADOS_LDFLAGS='-lrados' AC_SUBST(LIBRADOS_LDFLAGS) ][]) +AC_CHECK_LIB(s3, S3_deinitialize, [ + AC_DEFINE([HAVE_LIBS3], [1], [Have found libs3]) + LIBS3_LDFLAGS='-ls3' + AC_SUBST(LIBS3_LDFLAGS) +][]) AC_CHECK_FUNCS([bzero gethostbyname gettimeofday inet_ntoa memset mkdir pow select socket strchr strdup strerror strrchr strstr strtol strtoul utime],[],[AC_MSG_ERROR([One of the required functions was not found])]) AC_CHECK_FUNCS([daemon getloadavg],[],[]) diff --git a/includes/store_s3.h b/includes/store_s3.h new file mode 100644 index 00000000..362396d0 --- /dev/null +++ b/includes/store_s3.h @@ -0,0 +1,16 @@ +#ifndef STORES3_H +#define STORES3_H + +#ifdef __cplusplus +extern "C" +{ +#endif + +#include "store.h" + +struct storage_backend* init_storage_s3(const char *connection_string); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/src/store_s3.c b/src/store_s3.c index d1441784..8cdd5fc0 100644 --- a/src/store_s3.c +++ b/src/store_s3.c @@ -7,8 +7,6 @@ #include #include -#define HAVE_LIBS3 1 - #ifdef HAVE_LIBS3 #include #endif @@ -23,6 +21,19 @@ static pthread_mutex_t qLock; static int store_s3_initialized = 0; +struct s3_tile_request +{ + const char *path; + size_t tile_size; + int stat_only; + char *tile; + int64_t tile_mod_time; + int tile_expired; + size_t cur_offset; + S3Status result; + const S3ErrorDetails *error_details; +}; + struct tile_cache { struct stat_info st_stat; @@ -38,58 +49,127 @@ struct store_s3_ctx struct tile_cache cache; }; -struct MemoryStruct -{ - char *memory; - size_t size; -}; - static char* store_s3_xyz_to_storagekey(struct storage_backend *store, int x, int y, int z, char *key) { - snprintf(key, PATH_MAX - 1, "/%s/%i/%i/%i.png", ((struct store_s3_ctx *) (store->storage_ctx))->basepath, z, x, y); + snprintf(key, PATH_MAX - 1, "/%s/%i/%i/%i.png", ((struct store_s3_ctx*) (store->storage_ctx))->basepath, z, x, y); return key; } -static int store_s3_tile_retrieve(struct storage_backend * store, const char *xmlconfig, const char *options, int x, int y, int z) +static S3Status store_s3_properties_callback(const S3ResponseProperties *properties, void *callbackData) { - struct store_s3_ctx * ctx = (struct store_s3_ctx *) (store->storage_ctx); - char * path; + struct s3_tile_request *rqst = (struct s3_tile_request*) callbackData; + log_message(STORE_LOGLVL_DEBUG, "store_s3_properties_callback: got properties for tile, length: %ld, content type: %s", properties->contentLength, properties->contentType); + + rqst->tile_size = properties->contentLength; + rqst->tile_mod_time = properties->lastModified; + if (!rqst->stat_only) { + rqst->tile = malloc(properties->contentLength); + rqst->cur_offset = 0; + } + rqst->tile_expired = 0; + const S3NameValue *respMetadata = properties->metaData; + for (int i = 0; i < properties->metaDataCount; i++) { + if (0 == strcmp(respMetadata[i].name, "expired")) { + rqst->tile_expired = atoi(respMetadata[i].value); + } + } + + return S3StatusOK; +} + +S3Status store_s3_object_data_callback(int bufferSize, const char *buffer, void *callbackData) +{ + struct s3_tile_request *rqst = (struct s3_tile_request*) callbackData; + log_message(STORE_LOGLVL_DEBUG, "store_s3_object_data_callback: appending %ld bytes to buffer, new offset %ld", bufferSize, rqst->cur_offset + + bufferSize); + memcpy(rqst->tile + rqst->cur_offset, buffer, bufferSize); + rqst->cur_offset += bufferSize; + return S3StatusOK; +} + +int store_s3_put_object_data_callback(int bufferSize, char *buffer, void *callbackData) +{ + struct s3_tile_request *rqst = (struct s3_tile_request*) callbackData; + size_t bytesToWrite = MAX(bufferSize, rqst->tile_size - rqst->cur_offset); + log_message(STORE_LOGLVL_DEBUG, "store_s3_properties_callback: got properties for tile, writing %ld bytes to buffer, new offset %ld", bytesToWrite, rqst->cur_offset + + bytesToWrite); + memcpy(buffer, rqst->tile + rqst->cur_offset, bytesToWrite); + rqst->cur_offset += bytesToWrite; + int written = 0; + if (rqst->cur_offset == rqst->tile_size) { + // indicate "end of data" + written = 0; + } else { + written = bytesToWrite; + } + return written; +} + +void store_s3_complete_callback(S3Status status, const S3ErrorDetails *errorDetails, void *callbackData) +{ + struct s3_tile_request *rqst = (struct s3_tile_request*) callbackData; + log_message(STORE_LOGLVL_DEBUG, "store_s3_complete_callback: request complete, status %d", status); + if (errorDetails) { + log_message(STORE_LOGLVL_DEBUG, " error details: %s", errorDetails); + } + rqst->result = status; + rqst->error_details = errorDetails; +} + +static int store_s3_tile_retrieve(struct storage_backend *store, const char *xmlconfig, const char *options, int x, int y, int z) +{ + struct store_s3_ctx *ctx = (struct store_s3_ctx*) store->storage_ctx; + char *path = NULL; if ((ctx->cache.x == x) && (ctx->cache.y == y) && (ctx->cache.z == z) && (strcmp(ctx->cache.xmlname, xmlconfig) == 0)) { - log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_fetch: Got a cached tile"); + log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_retrieve: Got a cached tile"); return 1; } - log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_fetch: Fetching tile"); + log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_retrieve: Fetching tile"); path = malloc(PATH_MAX); store_s3_xyz_to_storagekey(store, x, y, z, path); - log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_fetch: retrieving file %s", path); + log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_retrieve: retrieving object %s", path); + + struct S3GetObjectHandler getObjectHandler; + getObjectHandler.responseHandler.propertiesCallback = + &store_s3_properties_callback; + getObjectHandler.responseHandler.completeCallback = + &store_s3_complete_callback; + getObjectHandler.getObjectDataCallback = &store_s3_object_data_callback; - S3Status res = S3_get_object(ctx->ctx, path, NULL, 0, 0, NULL, handler, NULL); + struct s3_tile_request request; + request.path = path; + request.stat_only = 0; + + S3_get_object(ctx->ctx, path, NULL, 0, 0, NULL, &getObjectHandler, &request); free(path); - if (res != S3StatusOK) { - log_message(STORE_LOGLVL_ERR, "store_s3_tile_fetch: failed to retrieve file: %s", S3_get_status_name(res)); + if (request.result != S3StatusOK) { + log_message(STORE_LOGLVL_ERR, "store_s3_tile_retrieve: failed to retrieve object: %d/%s", request.result, request.error_details); + if (ctx->cache.tile) { + free(ctx->cache.tile); + ctx->cache.tile = NULL; + } ctx->cache.x = -1; ctx->cache.y = -1; ctx->cache.z = -1; return -1; } - if (ctx->cache.tile != NULL) + log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_retrieve: Read object of size %i", request.tile_size); + + if (ctx->cache.tile) { free(ctx->cache.tile); - ctx->cache.tile = chunk.memory; - ctx->cache.st_stat.size = chunk.size; - ctx->cache.st_stat.expired = 0; - res = - curl_easy_getinfo(ctx->ctx, CURLINFO_FILETIME, &(ctx->cache.st_stat.mtime)); + } + ctx->cache.tile = request.tile; + ctx->cache.st_stat.size = request.tile_size; ctx->cache.st_stat.atime = 0; - log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_read: Read file of size %i", chunk.size); - break; - + ctx->cache.st_stat.mtime = request.tile_mod_time; + ctx->cache.st_stat.expired = 0; ctx->cache.x = x; ctx->cache.y = y; ctx->cache.z = z; @@ -97,31 +177,54 @@ static int store_s3_tile_retrieve(struct storage_backend * store, const char *xm return 1; } -static int store_s3_tile_read(struct storage_backend * store, const char *xmlconfig, const char *options, int x, int y, int z, char *buf, size_t sz, int * compressed, char * log_msg) +static int store_s3_tile_read(struct storage_backend *store, const char *xmlconfig, const char *options, int x, int y, int z, char *buf, size_t sz, int * compressed, char * log_msg) { - struct store_s3_ctx * ctx = (struct store_s3_ctx *) (store->storage_ctx); + struct store_s3_ctx *ctx = (struct store_s3_ctx*) store->storage_ctx; - if (store_s3_tile_retrieve(store, xmlconfig, options, x, y, z) > 0) { - if (ctx->cache.st_stat.size > sz) { - log_message(STORE_LOGLVL_ERR, "store_s3_tile_read: size was too big, overrun %i %i", sz, ctx->cache.st_stat.size); - return -1; - } - memcpy(buf, ctx->cache.tile, ctx->cache.st_stat.size); - return ctx->cache.st_stat.size; - } else { + if (store_s3_tile_retrieve(store, xmlconfig, options, x, y, z) <= 0) { log_message(STORE_LOGLVL_ERR, "store_s3_tile_read: Fetching didn't work"); return -1; } + if (ctx->cache.st_stat.size > sz) { + log_message(STORE_LOGLVL_ERR, "store_s3_tile_read: buffer not big enough for tile (%i < %i)", sz, ctx->cache.st_stat.size); + return -1; + } + memcpy(buf, ctx->cache.tile, ctx->cache.st_stat.size); + return ctx->cache.st_stat.size; } -static struct stat_info store_s3_tile_stat(struct storage_backend * store, const char *xmlconfig, const char *options, int x, int y, int z) +static struct stat_info store_s3_tile_stat(struct storage_backend *store, const char *xmlconfig, const char *options, int x, int y, int z) { - struct stat_info tile_stat; - struct store_s3_ctx * ctx = (struct store_s3_ctx *) (store->storage_ctx); + struct store_s3_ctx *ctx = (struct store_s3_ctx*) store->storage_ctx; + char *path = NULL; - if (store_s3_tile_retrieve(store, xmlconfig, options, x, y, z) > 0) { + if ((ctx->cache.x == x) && (ctx->cache.y == y) && (ctx->cache.z == z) + && (strcmp(ctx->cache.xmlname, xmlconfig) == 0)) { + log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_stat: Got a cached tile"); return ctx->cache.st_stat; - } else { + } + + log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_stat: Fetching tile properties"); + + path = malloc(PATH_MAX); + + store_s3_xyz_to_storagekey(store, x, y, z, path); + log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_stat: getting properties for object %s", path); + + struct S3ResponseHandler responseHandler; + responseHandler.propertiesCallback = &store_s3_properties_callback; + responseHandler.completeCallback = &store_s3_complete_callback; + + struct s3_tile_request request; + request.path = path; + request.stat_only = 1; + + S3_head_object(ctx->ctx, path, NULL, &responseHandler, &request); + free(path); + + struct stat_info tile_stat; + if (request.result != S3StatusOK) { + log_message(STORE_LOGLVL_ERR, "store_s3_tile_retrieve: failed to retrieve object properties: %d/%s", request.result, request.error_details); tile_stat.size = -1; tile_stat.expired = 0; tile_stat.mtime = 0; @@ -129,38 +232,125 @@ static struct stat_info store_s3_tile_stat(struct storage_backend * store, const tile_stat.ctime = 0; return tile_stat; } + + log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_retrieve: Read properties"); + + tile_stat.size = request.tile_size; + tile_stat.expired = request.tile_expired; + tile_stat.mtime = request.tile_mod_time; + tile_stat.atime = 0; + tile_stat.ctime = 0; + return tile_stat; } -static char * store_s3_tile_storage_id(struct storage_backend * store, const char *xmlconfig, const char *options, int x, int y, int z, char * string) +static char* store_s3_tile_storage_id(struct storage_backend *store, const char *xmlconfig, const char *options, int x, int y, int z, char * string) { - return store_s3_xyz_to_storagekey(store, x, y, z, string); } -static int store_s3_metatile_write(struct storage_backend * store, const char *xmlconfig, const char *options, int x, int y, int z, const char *buf, int sz) +static int store_s3_tile_write(struct storage_backend *store, const char *xmlconfig, const char *options, int x, int y, int z, const char *buf, int sz) { - log_message(STORE_LOGLVL_ERR, "store_s3_metatile_write: This is a readonly storage backend. Write functionality isn't implemented"); - return -1; + struct store_s3_ctx *ctx = (struct store_s3_ctx*) store->storage_ctx; + char *path = malloc(PATH_MAX); + store_s3_xyz_to_storagekey(store, x, y, z, path); + log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_write: storing object %s", path); + + struct S3PutObjectHandler putObjectHandler; + putObjectHandler.responseHandler.propertiesCallback = + &store_s3_properties_callback; + putObjectHandler.responseHandler.completeCallback = + &store_s3_complete_callback; + putObjectHandler.putObjectDataCallback = &store_s3_put_object_data_callback; + + struct s3_tile_request request; + request.path = path; + request.tile = buf; + request.tile_size = sz; + request.cur_offset = 0; + + S3_put_object(ctx->ctx, path, sz, NULL, NULL, &putObjectHandler, &request); + free(path); + + if (request.result != S3StatusOK) { + log_message(STORE_LOGLVL_ERR, "store_s3_tile_write: failed to write object: %d/%s", request.result, request.error_details); + return -1; + } + + log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_write: Wrote object of size %i", request.tile_size); + + return sz; } -static int store_s3_metatile_delete(struct storage_backend * store, const char *xmlconfig, int x, int y, int z) +static int store_s3_tile_delete(struct storage_backend *store, const char *xmlconfig, int x, int y, int z) { - log_message(STORE_LOGLVL_ERR, "store_s3_metatile_expire: This is a readonly storage backend. Write functionality isn't implemented"); - return -1; + struct store_s3_ctx *ctx = (struct store_s3_ctx*) store->storage_ctx; + char *path = malloc(PATH_MAX); + store_s3_xyz_to_storagekey(store, x, y, z, path); + log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_write: deleting object %s", path); + + struct S3ResponseHandler responseHandler; + responseHandler.propertiesCallback = &store_s3_properties_callback; + responseHandler.completeCallback = &store_s3_complete_callback; + + struct s3_tile_request request; + request.path = path; + request.stat_only = 1; + + S3_delete_object(ctx->ctx, path, NULL, &responseHandler, &request); + + if (request.result != S3StatusOK) { + log_message(STORE_LOGLVL_ERR, "store_s3_tile_delete: failed to delete object: %d/%s", request.result, request.error_details); + return -1; + } + + log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_delete: deleted object"); + + return 0; } -static int store_s3_metatile_expire(struct storage_backend *store, const char *xmlconfig, int x, int y, int z) +static int store_s3_tile_expire(struct storage_backend *store, const char *xmlconfig, int x, int y, int z) { + struct store_s3_ctx *ctx = (struct store_s3_ctx*) store->storage_ctx; + char *path = malloc(PATH_MAX); + store_s3_xyz_to_storagekey(store, x, y, z, path); + log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_expire: expiring object %s", path); + + struct S3ResponseHandler responseHandler; + responseHandler.propertiesCallback = &store_s3_properties_callback; + responseHandler.completeCallback = &store_s3_complete_callback; - log_message(STORE_LOGLVL_ERR, "store_s3_metatile_expire: This is a readonly storage backend. Write functionality isn't implemented"); - return -1; + struct s3_tile_request request; + request.path = path; + request.cur_offset = 0; + + struct S3NameValue expireTag; + expireTag.name = "expired"; + expireTag.value = "1"; + + struct S3PutProperties putProperties; + putProperties.metaDataCount = 1; + putProperties.metaData = &expireTag; + + int64_t lastModified; + + S3_copy_object(ctx->ctx, path, ctx->ctx->bucketName, path, &putProperties, &lastModified, 0, NULL, NULL, &responseHandler, &request); + free(path); + + if (request.result != S3StatusOK) { + log_message(STORE_LOGLVL_ERR, "store_s3_tile_expire: failed to update object: %d/%s", request.result, request.error_details); + return -1; + } + + log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_expire: Updated object metadata"); + + return 0; } static int store_s3_close_storage(struct storage_backend *store) { struct store_s3_ctx * ctx = (struct store_s3_ctx *) (store->storage_ctx); - free(ctx->baseurl); + free(ctx->basepath); if (ctx->cache.tile) free(ctx->cache.tile); S3_deinitialize(); @@ -170,7 +360,7 @@ static int store_s3_close_storage(struct storage_backend *store) return 0; } -#endif //Have curl +#endif //Have libs3 struct storage_backend * init_storage_s3(const char *connection_string) { @@ -232,9 +422,9 @@ struct storage_backend * init_storage_s3(const char *connection_string) store->tile_read = &store_s3_tile_read; store->tile_stat = &store_s3_tile_stat; - store->metatile_write = &store_s3_metatile_write; - store->metatile_delete = &store_s3_metatile_delete; - store->metatile_expire = &store_s3_metatile_expire; + store->metatile_write = &store_s3_tile_write; + store->metatile_delete = &store_s3_tile_delete; + store->metatile_expire = &store_s3_tile_expire; store->tile_storage_id = &store_s3_tile_storage_id; store->close_storage = &store_s3_close_storage; From 076038b5729c9790698f59118dd070712b216cf5 Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Thu, 14 Jan 2016 11:27:57 -0700 Subject: [PATCH 03/35] allow case where default host name for s3 access isn't specified in connection string --- src/store_s3.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/store_s3.c b/src/store_s3.c index 8cdd5fc0..fc6d49aa 100644 --- a/src/store_s3.c +++ b/src/store_s3.c @@ -362,7 +362,7 @@ static int store_s3_close_storage(struct storage_backend *store) #endif //Have libs3 -struct storage_backend * init_storage_s3(const char *connection_string) +struct storage_backend* init_storage_s3(const char *connection_string) { #ifndef HAVE_LIBS3 log_message(STORE_LOGLVL_ERR, @@ -393,7 +393,7 @@ struct storage_backend * init_storage_s3(const char *connection_string) pthread_mutex_lock(&qLock); if (!store_s3_initialized) { - log_message(STORE_LOGLVL_DEBUG, "init_storage_s3: Global init of curl", connection_string); + log_message(STORE_LOGLVL_DEBUG, "init_storage_s3: Global init of libs3", connection_string); res = S3_initialize(NULL, S3_INIT_ALL, NULL); store_s3_initialized = 1; } else { @@ -407,14 +407,19 @@ struct storage_backend * init_storage_s3(const char *connection_string) return NULL; } - // parse out the context information from the URL: s3://:@/[/] + // parse out the context information from the URL: s3://:[@]/[/] ctx->ctx = malloc(sizeof(struct S3BucketContext)); char *fullurl = strdup(connection_string); fullurl = &fullurl[5]; // advance past "s3://" ctx->ctx->accessKeyId = strsep(&fullurl, ":"); - ctx->ctx->secretAccessKey = strsep(&fullurl, "@"); - ctx->ctx->hostName = strsep(&fullurl, "/"); + if (strchr(fullurl, "@")) { + ctx->ctx->secretAccessKey = strsep(&fullurl, "@"); + ctx->ctx->hostName = strsep(&fullurl, "/"); + } + else { + ctx->ctx->secretAccessKey = strsep(&fullurl, "/"); + } ctx->ctx->bucketName = strsep(&fullurl, "/"); ctx->basepath = fullurl; From 62dea33ba1e68ff4206e322e4f88ee8bd97fd12a Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Thu, 14 Jan 2016 14:29:26 -0700 Subject: [PATCH 04/35] add url-decoding to s3 connection string --- src/store_s3.c | 43 +++++++++++++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/src/store_s3.c b/src/store_s3.c index fc6d49aa..4a12b042 100644 --- a/src/store_s3.c +++ b/src/store_s3.c @@ -360,6 +360,26 @@ static int store_s3_close_storage(struct storage_backend *store) return 0; } +static char* url_decode(const char *src) +{ + char *dst = (char*) malloc(strlen(src) + 1); + dst[0] = '\0'; + while (*src) { + int c = *src; + if (c == '%' && isxdigit(*(src + 1)) && isxdigit(*(src + 2))) { + char hexdigit[] = + { *(src + 1), *(src + 2), '\0' }; + char decodedchar[2]; + sprintf(decodedchar, "%c", (char) strtol(hexdigit, NULL, 16)); + strncat(dst, decodedchar, 1); + src += 2; + } else { + strncat(dst, src, 1); + } + src++; + } + return dst; +} #endif //Have libs3 struct storage_backend* init_storage_s3(const char *connection_string) @@ -411,17 +431,20 @@ struct storage_backend* init_storage_s3(const char *connection_string) ctx->ctx = malloc(sizeof(struct S3BucketContext)); char *fullurl = strdup(connection_string); - fullurl = &fullurl[5]; // advance past "s3://" - ctx->ctx->accessKeyId = strsep(&fullurl, ":"); - if (strchr(fullurl, "@")) { - ctx->ctx->secretAccessKey = strsep(&fullurl, "@"); - ctx->ctx->hostName = strsep(&fullurl, "/"); - } - else { - ctx->ctx->secretAccessKey = strsep(&fullurl, "/"); + // advance past "s3://" + fullurl = &fullurl[5]; + ctx->ctx->accessKeyId = url_decode(strsep(&fullurl, ":")); + if (strchr(fullurl, '@')) { + ctx->ctx->secretAccessKey = url_decode(strsep(&fullurl, "@")); + ctx->ctx->hostName = url_decode(strsep(&fullurl, "/")); + } else { + ctx->ctx->secretAccessKey = url_decode(strsep(&fullurl, "/")); } - ctx->ctx->bucketName = strsep(&fullurl, "/"); - ctx->basepath = fullurl; + ctx->ctx->bucketName = url_decode(strsep(&fullurl, "/")); + + ctx->basepath = url_decode(fullurl); + + log_message(STORE_LOGLVL_DEBUG, "init_storage_s3 completed keyid: %s, key: %s, bucket: %s, basepath: %s", ctx->ctx->accessKeyId, ctx->ctx->secretAccessKey, ctx->ctx->bucketName, ctx->basepath); store->storage_ctx = ctx; From 0533c16e103255d6387cd2093a168599c0c10a5f Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Thu, 14 Jan 2016 14:57:55 -0700 Subject: [PATCH 05/35] add url detection for s3 storage --- src/store.c | 6 ++++++ src/store_s3.c | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/store.c b/src/store.c index f9d511c7..243cbb23 100644 --- a/src/store.c +++ b/src/store.c @@ -21,6 +21,7 @@ #include "store_rados.h" #include "store_ro_http_proxy.h" #include "store_ro_composite.h" +#include "store_s3.h" #include "store_null.h" //TODO: Make this function handle different logging backends, depending on if on compiles it from apache or something else @@ -103,6 +104,11 @@ struct storage_backend * init_storage_backend(const char * options) { store = init_storage_ro_composite(options); return store; } + if (strstr(options, "s3://") == options) { + log_message(STORE_LOGLVL_DEBUG, "init_storage_backend: initialising s3 storage backend at: %s", options); + store = init_storage_s3(options); + return store; + } if (strstr(options,"null://") == options) { log_message(STORE_LOGLVL_DEBUG, "init_storage_backend: initialising null storage backend at: %s", options); store = init_storage_null(); diff --git a/src/store_s3.c b/src/store_s3.c index 4a12b042..0f950d9a 100644 --- a/src/store_s3.c +++ b/src/store_s3.c @@ -1,4 +1,5 @@ #include "config.h" +#include #include #include #include @@ -264,7 +265,7 @@ static int store_s3_tile_write(struct storage_backend *store, const char *xmlcon struct s3_tile_request request; request.path = path; - request.tile = buf; + request.tile = (char*) buf; request.tile_size = sz; request.cur_offset = 0; @@ -350,7 +351,6 @@ static int store_s3_close_storage(struct storage_backend *store) { struct store_s3_ctx * ctx = (struct store_s3_ctx *) (store->storage_ctx); - free(ctx->basepath); if (ctx->cache.tile) free(ctx->cache.tile); S3_deinitialize(); From cfbc06e123a830e983af4e6d54f04ab067f8d8aa Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Thu, 14 Jan 2016 15:06:52 -0700 Subject: [PATCH 06/35] add new storage module to apache module build --- Makefile.am | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile.am b/Makefile.am index 360a0f58..a836930b 100644 --- a/Makefile.am +++ b/Makefile.am @@ -48,10 +48,10 @@ test: gen_tile_test ./gen_tile_test all-local: - $(APXS) -c $(DEF_LDLIBS) $(AM_CFLAGS) -I@srcdir@/includes $(AM_LDFLAGS) $(STORE_LDFLAGS) @srcdir@/src/mod_tile.c @srcdir@/src/sys_utils.c @srcdir@/src/store.c @srcdir@/src/store_file.c @srcdir@/src/store_file_utils.c @srcdir@/src/store_memcached.c @srcdir@/src/store_rados.c @srcdir@/src/store_ro_http_proxy.c @srcdir@/src/store_ro_composite.c @srcdir@/src/store_null.c + $(APXS) -c $(DEF_LDLIBS) $(AM_CFLAGS) -I@srcdir@/includes $(AM_LDFLAGS) $(STORE_LDFLAGS) @srcdir@/src/mod_tile.c @srcdir@/src/sys_utils.c @srcdir@/src/store.c @srcdir@/src/store_file.c @srcdir@/src/store_file_utils.c @srcdir@/src/store_memcached.c @srcdir@/src/store_rados.c @srcdir@/src/store_ro_http_proxy.c @srcdir@/src/store_ro_composite.c @srcdir@/src/store_s3.c @srcdir@/src/store_null.c install-mod_tile: mkdir -p $(DESTDIR)`$(APXS) -q LIBEXECDIR` - $(APXS) -S LIBEXECDIR=$(DESTDIR)`$(APXS) -q LIBEXECDIR` -c -i $(DEF_LDLIBS) $(AM_CFLAGS) -I@srcdir@/includes $(AM_LDFLAGS) $(STORE_LDFLAGS) @srcdir@/src/mod_tile.c @srcdir@/src/sys_utils.c @srcdir@/src/store.c @srcdir@/src/store_file.c @srcdir@/src/store_file_utils.c @srcdir@/src/store_memcached.c @srcdir@/src/store_rados.c @srcdir@/src/store_ro_http_proxy.c @srcdir@/src/store_ro_composite.c @srcdir@/src/store_null.c + $(APXS) -S LIBEXECDIR=$(DESTDIR)`$(APXS) -q LIBEXECDIR` -c -i $(DEF_LDLIBS) $(AM_CFLAGS) -I@srcdir@/includes $(AM_LDFLAGS) $(STORE_LDFLAGS) @srcdir@/src/mod_tile.c @srcdir@/src/sys_utils.c @srcdir@/src/store.c @srcdir@/src/store_file.c @srcdir@/src/store_file_utils.c @srcdir@/src/store_memcached.c @srcdir@/src/store_rados.c @srcdir@/src/store_ro_http_proxy.c @srcdir@/src/store_ro_composite.c @srcdir@/src/store_s3.c @srcdir@/src/store_null.c From a099f938c81ffcc045718a76571cc8945b2efa36 Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Thu, 14 Jan 2016 15:24:03 -0700 Subject: [PATCH 07/35] fix error logging --- src/store_s3.c | 41 +++++++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/src/store_s3.c b/src/store_s3.c index 0f950d9a..875b1f53 100644 --- a/src/store_s3.c +++ b/src/store_s3.c @@ -109,9 +109,10 @@ int store_s3_put_object_data_callback(int bufferSize, char *buffer, void *callba void store_s3_complete_callback(S3Status status, const S3ErrorDetails *errorDetails, void *callbackData) { struct s3_tile_request *rqst = (struct s3_tile_request*) callbackData; - log_message(STORE_LOGLVL_DEBUG, "store_s3_complete_callback: request complete, status %d", status); - if (errorDetails) { - log_message(STORE_LOGLVL_DEBUG, " error details: %s", errorDetails); + log_message(STORE_LOGLVL_DEBUG, "store_s3_complete_callback: request complete, status %d (%s)", status, S3_get_status_name(status)); + if (errorDetails && errorDetails->message + && (strlen(errorDetails->message) > 0)) { + log_message(STORE_LOGLVL_DEBUG, " error details: %s", errorDetails); } rqst->result = status; rqst->error_details = errorDetails; @@ -150,7 +151,11 @@ static int store_s3_tile_retrieve(struct storage_backend *store, const char *xml free(path); if (request.result != S3StatusOK) { - log_message(STORE_LOGLVL_ERR, "store_s3_tile_retrieve: failed to retrieve object: %d/%s", request.result, request.error_details); + const char *msg = ""; + if (request.error_details && request.error_details->message) { + msg = request.error_details->message; + } + log_message(STORE_LOGLVL_ERR, "store_s3_tile_retrieve: failed to retrieve object: %d(%s)/%s", request.result, S3_get_status_name(request.result), msg); if (ctx->cache.tile) { free(ctx->cache.tile); ctx->cache.tile = NULL; @@ -183,7 +188,7 @@ static int store_s3_tile_read(struct storage_backend *store, const char *xmlconf struct store_s3_ctx *ctx = (struct store_s3_ctx*) store->storage_ctx; if (store_s3_tile_retrieve(store, xmlconfig, options, x, y, z) <= 0) { - log_message(STORE_LOGLVL_ERR, "store_s3_tile_read: Fetching didn't work"); + log_message(STORE_LOGLVL_ERR, "store_s3_tile_read: retrieval failed"); return -1; } if (ctx->cache.st_stat.size > sz) { @@ -201,7 +206,7 @@ static struct stat_info store_s3_tile_stat(struct storage_backend *store, const if ((ctx->cache.x == x) && (ctx->cache.y == y) && (ctx->cache.z == z) && (strcmp(ctx->cache.xmlname, xmlconfig) == 0)) { - log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_stat: Got a cached tile"); + log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_stat: got a cached tile"); return ctx->cache.st_stat; } @@ -225,7 +230,11 @@ static struct stat_info store_s3_tile_stat(struct storage_backend *store, const struct stat_info tile_stat; if (request.result != S3StatusOK) { - log_message(STORE_LOGLVL_ERR, "store_s3_tile_retrieve: failed to retrieve object properties: %d/%s", request.result, request.error_details); + const char *msg = ""; + if (request.error_details && request.error_details->message) { + msg = request.error_details->message; + } + log_message(STORE_LOGLVL_ERR, "store_s3_tile_retrieve: failed to retrieve object properties: %d(%s)/%s", request.result, S3_get_status_name(request.result), msg); tile_stat.size = -1; tile_stat.expired = 0; tile_stat.mtime = 0; @@ -273,7 +282,11 @@ static int store_s3_tile_write(struct storage_backend *store, const char *xmlcon free(path); if (request.result != S3StatusOK) { - log_message(STORE_LOGLVL_ERR, "store_s3_tile_write: failed to write object: %d/%s", request.result, request.error_details); + const char *msg = ""; + if (request.error_details && request.error_details->message) { + msg = request.error_details->message; + } + log_message(STORE_LOGLVL_ERR, "store_s3_tile_write: failed to write object: %d(%s)/%s", request.result, S3_get_status_name(request.result), msg); return -1; } @@ -300,7 +313,11 @@ static int store_s3_tile_delete(struct storage_backend *store, const char *xmlco S3_delete_object(ctx->ctx, path, NULL, &responseHandler, &request); if (request.result != S3StatusOK) { - log_message(STORE_LOGLVL_ERR, "store_s3_tile_delete: failed to delete object: %d/%s", request.result, request.error_details); + const char *msg = ""; + if (request.error_details && request.error_details->message) { + msg = request.error_details->message; + } + log_message(STORE_LOGLVL_ERR, "store_s3_tile_delete: failed to delete object: %d(%s)/%s", request.result, S3_get_status_name(request.result), msg); return -1; } @@ -338,7 +355,11 @@ static int store_s3_tile_expire(struct storage_backend *store, const char *xmlco free(path); if (request.result != S3StatusOK) { - log_message(STORE_LOGLVL_ERR, "store_s3_tile_expire: failed to update object: %d/%s", request.result, request.error_details); + const char *msg = ""; + if (request.error_details && request.error_details->message) { + msg = request.error_details->message; + } + log_message(STORE_LOGLVL_ERR, "store_s3_tile_expire: failed to update object: %d(%s)/%s", request.result, S3_get_status_name(request.result), msg); return -1; } From f02b617a88f096f870c56a79112264bc89fa197b Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Tue, 19 Jan 2016 11:04:48 -0700 Subject: [PATCH 08/35] added s3 testing to unit test, got s3 working with metatiles --- includes/metatile.h | 29 ++-- includes/store_file.h | 5 +- includes/store_file_utils.h | 1 + src/gen_tile_test.cpp | 300 ++++++++++++++++++++++++++++++++++ src/metatile.cpp | 1 - src/store_file.c | 9 +- src/store_s3.c | 310 +++++++++++++++++++----------------- 7 files changed, 490 insertions(+), 165 deletions(-) diff --git a/includes/metatile.h b/includes/metatile.h index c5c8d569..3cf74afa 100644 --- a/includes/metatile.h +++ b/includes/metatile.h @@ -12,21 +12,22 @@ extern "C" { #define META_MAGIC "META" #define META_MAGIC_COMPRESSED "METZ" - - struct entry { - int offset; - int size; - }; - - struct meta_layout { - char magic[4]; - int count; // METATILE ^ 2 - int x, y, z; // lowest x,y of this metatile, plus z - struct entry index[]; // count entries - // Followed by the tile data - // The index offsets are measured from the start of the file - }; +struct entry { + int offset; + int size; +}; + +struct meta_layout { + char magic[4]; + int count; // METATILE ^ 2 + int x, y, z; // lowest x,y of this metatile, plus z + struct entry index[]; // count entries + // Followed by the tile data + // The index offsets are measured from the start of the file +}; + +#define METATILE_HEADER_LEN (sizeof(struct meta_layout) + METATILE * METATILE * sizeof(struct entry)) #ifdef __cplusplus } diff --git a/includes/store_file.h b/includes/store_file.h index 77caae51..faa44aa6 100644 --- a/includes/store_file.h +++ b/includes/store_file.h @@ -6,9 +6,8 @@ extern "C" { #endif #include "store.h" - - struct storage_backend * init_storage_file(const char * tile_dir); - int xyzo_to_meta(char *path, size_t len, const char *tile_dir, const char *xmlconfig, const char *options, int x, int y, int z); + +struct storage_backend * init_storage_file(const char * tile_dir); #ifdef __cplusplus } diff --git a/includes/store_file_utils.h b/includes/store_file_utils.h index 40e167b1..dfef0fa8 100644 --- a/includes/store_file_utils.h +++ b/includes/store_file_utils.h @@ -27,6 +27,7 @@ int path_to_xyz(const char *tilepath, const char *path, char *xmlconfig, int *px /* New meta-tile storage functions */ /* Returns the path to the meta-tile and the offset within the meta-tile */ int xyz_to_meta(char *path, size_t len, const char *tile_dir, const char *xmlconfig, int x, int y, int z); +int xyzo_to_meta(char *path, size_t len, const char *tile_dir, const char *xmlconfig, const char *options, int x, int y, int z); #endif #ifdef __cplusplus diff --git a/src/gen_tile_test.cpp b/src/gen_tile_test.cpp index 903f8b0f..f306746e 100644 --- a/src/gen_tile_test.cpp +++ b/src/gen_tile_test.cpp @@ -58,6 +58,9 @@ #include #endif +#ifdef HAVE_LIBS3 +#include +#endif #define NO_QUEUE_REQUESTS 9 #define NO_TEST_REPEATS 100 @@ -946,6 +949,303 @@ TEST_CASE( "projections", "Test projections" ) { } } +#ifdef HAVE_LIBS3 +S3Status test_s3_properties_callback(const S3ResponseProperties *properties, void *callbackData) +{ + return S3StatusOK; +} + +void test_s3_complete_callback(S3Status status, const S3ErrorDetails *errorDetails, void *callbackData) +{ +} + + +TEST_CASE("storage-backend/s3", "S3 tile storage backend") { + + /* Setting up S3 location for testing */ + char *s3_connection_url; + char *keyid = getenv("S3_ACCESS_KEY_ID"); + REQUIRE(keyid != NULL); + char *accesskey = getenv("S3_SECRET_ACCESS_KEY"); + REQUIRE(accesskey != NULL); + char *bucketname = getenv("S3_BUCKET_NAME"); + REQUIRE(bucketname != NULL); + const char *bucketpath = "mod_tile_test"; + + s3_connection_url = (char*) malloc(1024); + sprintf(s3_connection_url, "s3://%s:%s/%s/%s", keyid, accesskey, bucketname, bucketpath); + + SECTION("storage-backend/s3/initialise", "should return 1") { + struct storage_backend *store = NULL; + store = init_storage_backend(s3_connection_url); + REQUIRE(store != NULL); + store->close_storage(store); + } + + SECTION("storage-backend/s3/stat/non existent", "should return size < 0") { + struct storage_backend *store = NULL; + struct stat_info sinfo; + + store = init_storage_backend(s3_connection_url); + REQUIRE(store != NULL); + + sinfo = store->tile_stat(store, "default", "", 0, 0, 0); + REQUIRE(sinfo.size < 0); + store->close_storage(store); + } + + SECTION("storage-backend/s3/read/non existent", "should return size < 0") { + struct storage_backend *store = NULL; + int size; + char *buf = (char*) malloc(10000); + int compressed; + char *err_msg = (char*) malloc(10000); + + store = init_storage_backend(s3_connection_url); + REQUIRE(store != NULL); + + size = store->tile_read(store, "default", "", 0, 0, 0, buf, 10000, &compressed, err_msg); + REQUIRE(size < 0); + + store->close_storage(store); + free(buf); + free(err_msg); + } + + SECTION("storage-backend/s3/write/full metatile", "should complete") { + struct storage_backend *store = NULL; + + store = init_storage_backend(s3_connection_url); + REQUIRE(store != NULL); + + metaTile tiles("default", "", 1024, 1024, 10); + for (int yy = 0; yy < METATILE; yy++) { + for (int xx = 0; xx < METATILE; xx++) { + std::string tile_data = "DEADBEEF"; + tiles.set(xx, yy, tile_data); + } + } + tiles.save(store); + + store->close_storage(store); + } + + SECTION("storage-backend/s3/stat/full metatile", "should complete") { + struct storage_backend *store = NULL; + struct stat_info sinfo; + + time_t before_write, after_write; + + store = init_storage_backend(s3_connection_url); + REQUIRE(store != NULL); + + metaTile tiles("default", "", 1024 + METATILE, 1024, 10); + time(&before_write); + for (int yy = 0; yy < METATILE; yy++) { + for (int xx = 0; xx < METATILE; xx++) { + std::string tile_data = "DEADBEEF"; + tiles.set(xx, yy, tile_data); + } + } + tiles.save(store); + sleep(1); + time(&after_write); + + for (int yy = 0; yy < METATILE; yy++) { + for (int xx = 0; xx < METATILE; xx++) { + sinfo = store->tile_stat(store, "default", "", 1024 + METATILE + yy, 1024 + xx, 10); + REQUIRE(sinfo.size > 0); + REQUIRE(sinfo.expired == 0); + REQUIRE(sinfo.mtime >= before_write); + REQUIRE(sinfo.mtime <= after_write); + } + } + + store->close_storage(store); + } + + SECTION("storage-backend/s3/read/full metatile", "should complete") { + std::cerr << "storage-backend/s3/read/full metatile" << std::endl; + + struct storage_backend *store = NULL; + char *buf; + char *buf_tmp; + char msg[4096]; + int compressed; + int tile_size; + + buf = (char*) malloc(8196); + buf_tmp = (char*) malloc(8196); + + time_t before_write, after_write; + + store = init_storage_backend(s3_connection_url); + REQUIRE(store != NULL); + + metaTile tiles("default", "", 1024 + METATILE, 1024, 10); + time(&before_write); + for (int yy = 0; yy < METATILE; yy++) { + for (int xx = 0; xx < METATILE; xx++) { + sprintf(buf, "DEADBEEF %i %i", xx, yy); + std::string tile_data(buf); + tiles.set(xx, yy, tile_data); + } + } + tiles.save(store); + time(&after_write); + + for (int yy = 0; yy < METATILE; yy++) { + for (int xx = 0; xx < METATILE; xx++) { + tile_size = store->tile_read(store, "default", "", 1024 + METATILE + xx, 1024 + yy, 10, buf, 8195, &compressed, msg); + REQUIRE(tile_size == 12); + sprintf(buf_tmp, "DEADBEEF %i %i", xx, yy); + REQUIRE(memcmp(buf_tmp, buf, 11) == 0); + } + } + + free(buf); + free(buf_tmp); + store->close_storage(store); + } + + SECTION("storage-backend/s3/read/partial metatile", "should return correct data") { + std::cerr << "storage-backend/s3/read/partial metatile" << std::endl; + + struct storage_backend *store = NULL; + char *buf; + char *buf_tmp; + char msg[4096]; + int compressed; + int tile_size; + + buf = (char*) malloc(8196); + buf_tmp = (char*) malloc(8196); + + time_t before_write, after_write; + + store = init_storage_backend(s3_connection_url); + REQUIRE(store != NULL); + + metaTile tiles("default", "", 1024 + 2*METATILE, 1024, 10); + time(&before_write); + for (int yy = 0; yy < METATILE; yy++) { + for (int xx = 0; xx < (METATILE >> 1); xx++) { + sprintf(buf, "DEADBEEF %i %i", xx, yy); + std::string tile_data(buf); + tiles.set(xx, yy, tile_data); + } + } + tiles.save(store); + time(&after_write); + + for (int yy = 0; yy < METATILE; yy++) { + for (int xx = 0; xx < METATILE; xx++) { + tile_size = store->tile_read(store, "default", "", 1024 + 2*METATILE + xx, 1024 + yy, 10, buf, 8195, &compressed, msg); + if (xx >= (METATILE >> 1)) { + REQUIRE (tile_size == 0); + } else { + REQUIRE (tile_size == 12); + sprintf(buf_tmp, "DEADBEEF %i %i", xx, yy); + REQUIRE (memcmp(buf_tmp, buf, 11) == 0); + } + } + } + + free(buf); + free(buf_tmp); + store->close_storage(store); + } + + SECTION("storage-backend/s3/delete metatile", "should delete tile from storage") { + struct storage_backend *store = NULL; + struct stat_info sinfo; + char *buf; + char *buf_tmp; + + buf = (char*) malloc(8196); + buf_tmp = (char*) malloc(8196); + + store = init_storage_backend(s3_connection_url); + REQUIRE(store != NULL); + + metaTile tiles("default", "", 1024 + 3*METATILE, 1024, 10); + + for (int yy = 0; yy < METATILE; yy++) { + for (int xx = 0; xx < METATILE; xx++) { + sprintf(buf, "DEADBEEF %i %i", xx, yy); + std::string tile_data(buf); + tiles.set(xx, yy, tile_data); + } + } + tiles.save(store); + + sinfo = store->tile_stat(store, "default", "", 1024 + 3*METATILE, 1024, 10); + + REQUIRE(sinfo.size > 0); + + store->metatile_delete(store, "default", 1024 + 3*METATILE, 1024, 10); + + sinfo = store->tile_stat(store, "default", "", 1024 + 3*METATILE, 1024, 10); + + REQUIRE(sinfo.size < 0); + + free(buf); + free(buf_tmp); + store->close_storage(store); + } + + SECTION("storage-backend/s3/expire/expiremetatile", "should expire the tile") { + struct storage_backend *store = NULL; + struct stat_info sinfo; + char *buf; + char *buf_tmp; + + buf = (char*) malloc(8196); + buf_tmp = (char*) malloc(8196); + + store = init_storage_backend(s3_connection_url); + REQUIRE(store != NULL); + + metaTile tiles("default", "", 1024 + 4*METATILE, 1024, 10); + + for (int yy = 0; yy < METATILE; yy++) { + for (int xx = 0; xx < METATILE; xx++) { + sprintf(buf, "DEADBEEF %i %i", xx, yy); + std::string tile_data(buf); + tiles.set(xx, yy, tile_data); + } + } + tiles.save(store); + + sinfo = store->tile_stat(store, "default", "", 1024 + 4*METATILE, 1024, 10); + + REQUIRE(sinfo.size > 0); + + store->metatile_expire(store, "default", 1024 + 4*METATILE, 1024, 10); + + sinfo = store->tile_stat(store, "default", "", 1024 + 4*METATILE, 1024, 10); + + REQUIRE(sinfo.size > 0); + REQUIRE(sinfo.expired > 0); + + free(buf); + free(buf_tmp); + store->close_storage(store); + } + + S3BucketContext ctx; + ctx.accessKeyId = keyid; + ctx.secretAccessKey = accesskey; + ctx.bucketName = bucketname; + ctx.hostName = NULL; + S3ResponseHandler handler; + handler.completeCallback = &test_s3_complete_callback; + handler.propertiesCallback = &test_s3_properties_callback; + S3_delete_object(&ctx, bucketpath, NULL, &handler, NULL); +} + +#endif + int main (int argc, char* const argv[]) { //std::ios_base::sync_with_stdio(false); diff --git a/src/metatile.cpp b/src/metatile.cpp index c0b55455..1fa3e6d5 100644 --- a/src/metatile.cpp +++ b/src/metatile.cpp @@ -32,7 +32,6 @@ #include "cache_expire.h" #include "request_queue.h" - metaTile::metaTile(const std::string &xmlconfig, const std::string &options, int x, int y, int z): x_(x), y_(y), z_(z), xmlconfig_(xmlconfig), options_(options) { clear(); diff --git a/src/store_file.c b/src/store_file.c index 9c824455..f574c2a3 100644 --- a/src/store_file.c +++ b/src/store_file.c @@ -49,8 +49,7 @@ static int file_tile_read(struct storage_backend * store, const char *xmlconfig, char path[PATH_MAX]; int meta_offset, fd; unsigned int pos; - unsigned int header_len = sizeof(struct meta_layout) + METATILE*METATILE*sizeof(struct entry); - struct meta_layout *m = (struct meta_layout *)malloc(header_len); + struct meta_layout *m = (struct meta_layout *)malloc(METATILE_HEADER_LEN); size_t file_offset, tile_size; meta_offset = xyzo_to_meta(path, sizeof(path), store->storage_ctx, xmlconfig, options, x, y, z); @@ -63,8 +62,8 @@ static int file_tile_read(struct storage_backend * store, const char *xmlconfig, } pos = 0; - while (pos < header_len) { - size_t len = header_len - pos; + while (pos < METATILE_HEADER_LEN) { + size_t len = METATILE_HEADER_LEN - pos; int got = read(fd, ((unsigned char *) m) + pos, len); if (got < 0) { snprintf(log_msg,PATH_MAX - 1, "Failed to read complete header for metatile %s Reason: %s\n", path, strerror(errno)); @@ -77,7 +76,7 @@ static int file_tile_read(struct storage_backend * store, const char *xmlconfig, break; } } - if (pos < header_len) { + if (pos < METATILE_HEADER_LEN) { snprintf(log_msg,PATH_MAX - 1, "Meta file %s too small to contain header\n", path); close(fd); free(m); diff --git a/src/store_s3.c b/src/store_s3.c index 875b1f53..bc8a4d2f 100644 --- a/src/store_s3.c +++ b/src/store_s3.c @@ -13,7 +13,9 @@ #endif #include "store.h" +#include "store_file_utils.h" #include "store_s3.h" +#include "metatile.h" #include "render_config.h" #include "protocol.h" @@ -22,11 +24,9 @@ static pthread_mutex_t qLock; static int store_s3_initialized = 0; -struct s3_tile_request -{ +struct s3_tile_request { const char *path; size_t tile_size; - int stat_only; char *tile; int64_t tile_mod_time; int tile_expired; @@ -35,38 +35,29 @@ struct s3_tile_request const S3ErrorDetails *error_details; }; -struct tile_cache -{ - struct stat_info st_stat; - char *tile; - int x, y, z; - char xmlname[XMLCONFIG_MAX]; -}; - -struct store_s3_ctx -{ +struct store_s3_ctx { S3BucketContext* ctx; const char *basepath; - struct tile_cache cache; }; -static char* store_s3_xyz_to_storagekey(struct storage_backend *store, int x, int y, int z, char *key) +static int store_s3_xyz_to_storagekey(struct storage_backend *store, const char *xmlconfig, const char *options, int x, int y, int z, char *key, size_t keylen) { - snprintf(key, PATH_MAX - 1, "/%s/%i/%i/%i.png", ((struct store_s3_ctx*) (store->storage_ctx))->basepath, z, x, y); - return key; + int offset; + if (options) { + offset = xyzo_to_meta(key, keylen, ((struct store_s3_ctx*) (store->storage_ctx))->basepath, xmlconfig, options, x, y, z); + } else { + offset = xyz_to_meta(key, keylen, ((struct store_s3_ctx*) (store->storage_ctx))->basepath, xmlconfig, x, y, z); + } + + return offset; } static S3Status store_s3_properties_callback(const S3ResponseProperties *properties, void *callbackData) { struct s3_tile_request *rqst = (struct s3_tile_request*) callbackData; - log_message(STORE_LOGLVL_DEBUG, "store_s3_properties_callback: got properties for tile, length: %ld, content type: %s", properties->contentLength, properties->contentType); rqst->tile_size = properties->contentLength; rqst->tile_mod_time = properties->lastModified; - if (!rqst->stat_only) { - rqst->tile = malloc(properties->contentLength); - rqst->cur_offset = 0; - } rqst->tile_expired = 0; const S3NameValue *respMetadata = properties->metaData; for (int i = 0; i < properties->metaDataCount; i++) { @@ -75,14 +66,21 @@ static S3Status store_s3_properties_callback(const S3ResponseProperties *propert } } + log_message(STORE_LOGLVL_DEBUG, "store_s3_properties_callback: got properties for tile %s, length: %ld, content type: %s, expired: %d", rqst->path, rqst->tile_size, properties->contentType, rqst->tile_expired); + return S3StatusOK; } S3Status store_s3_object_data_callback(int bufferSize, const char *buffer, void *callbackData) { struct s3_tile_request *rqst = (struct s3_tile_request*) callbackData; - log_message(STORE_LOGLVL_DEBUG, "store_s3_object_data_callback: appending %ld bytes to buffer, new offset %ld", bufferSize, rqst->cur_offset - + bufferSize); + + if (rqst->cur_offset == 0 && rqst->tile == NULL) { + log_message(STORE_LOGLVL_DEBUG, "store_s3_object_data_callback: allocating %ld byte buffer for tile", rqst->tile_size); + rqst->tile = malloc(rqst->tile_size); + } + + log_message(STORE_LOGLVL_DEBUG, "store_s3_object_data_callback: appending %ld bytes to buffer, new offset %ld", bufferSize, rqst->cur_offset + bufferSize); memcpy(rqst->tile + rqst->cur_offset, buffer, bufferSize); rqst->cur_offset += bufferSize; return S3StatusOK; @@ -91,112 +89,108 @@ S3Status store_s3_object_data_callback(int bufferSize, const char *buffer, void int store_s3_put_object_data_callback(int bufferSize, char *buffer, void *callbackData) { struct s3_tile_request *rqst = (struct s3_tile_request*) callbackData; - size_t bytesToWrite = MAX(bufferSize, rqst->tile_size - rqst->cur_offset); - log_message(STORE_LOGLVL_DEBUG, "store_s3_properties_callback: got properties for tile, writing %ld bytes to buffer, new offset %ld", bytesToWrite, rqst->cur_offset - + bytesToWrite); - memcpy(buffer, rqst->tile + rqst->cur_offset, bytesToWrite); - rqst->cur_offset += bytesToWrite; - int written = 0; if (rqst->cur_offset == rqst->tile_size) { // indicate "end of data" - written = 0; - } else { - written = bytesToWrite; + log_message(STORE_LOGLVL_DEBUG, "store_s3_put_object_data_callback: completed put"); + return 0; } - return written; + size_t bytesToWrite = MAX(bufferSize, rqst->tile_size - rqst->cur_offset); + log_message(STORE_LOGLVL_DEBUG, "store_s3_put_object_data_callback: uploading data, writing %ld bytes to buffer, cur offset %ld, new offset %ld", bytesToWrite, rqst->cur_offset, rqst->cur_offset + bytesToWrite); + memcpy(buffer, rqst->tile + rqst->cur_offset, bytesToWrite); + rqst->cur_offset += bytesToWrite; + return bytesToWrite; } void store_s3_complete_callback(S3Status status, const S3ErrorDetails *errorDetails, void *callbackData) { struct s3_tile_request *rqst = (struct s3_tile_request*) callbackData; log_message(STORE_LOGLVL_DEBUG, "store_s3_complete_callback: request complete, status %d (%s)", status, S3_get_status_name(status)); - if (errorDetails && errorDetails->message - && (strlen(errorDetails->message) > 0)) { - log_message(STORE_LOGLVL_DEBUG, " error details: %s", errorDetails); + if (errorDetails && errorDetails->message && (strlen(errorDetails->message) > 0)) { + log_message(STORE_LOGLVL_DEBUG, " error details: %s", errorDetails->message); } rqst->result = status; rqst->error_details = errorDetails; } -static int store_s3_tile_retrieve(struct storage_backend *store, const char *xmlconfig, const char *options, int x, int y, int z) +static int store_s3_tile_read(struct storage_backend *store, const char *xmlconfig, const char *options, int x, int y, int z, char *buf, size_t sz, int *compressed, char *log_msg) { struct store_s3_ctx *ctx = (struct store_s3_ctx*) store->storage_ctx; - char *path = NULL; - - if ((ctx->cache.x == x) && (ctx->cache.y == y) && (ctx->cache.z == z) - && (strcmp(ctx->cache.xmlname, xmlconfig) == 0)) { - log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_retrieve: Got a cached tile"); - return 1; - } - - log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_retrieve: Fetching tile"); + char *path = malloc(PATH_MAX); - path = malloc(PATH_MAX); + log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_retrieve: fetching tile"); - store_s3_xyz_to_storagekey(store, x, y, z, path); + int tile_offset = store_s3_xyz_to_storagekey(store, xmlconfig, options, x, y, z, path, PATH_MAX); log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_retrieve: retrieving object %s", path); struct S3GetObjectHandler getObjectHandler; - getObjectHandler.responseHandler.propertiesCallback = - &store_s3_properties_callback; - getObjectHandler.responseHandler.completeCallback = - &store_s3_complete_callback; + getObjectHandler.responseHandler.propertiesCallback = &store_s3_properties_callback; + getObjectHandler.responseHandler.completeCallback = &store_s3_complete_callback; getObjectHandler.getObjectDataCallback = &store_s3_object_data_callback; struct s3_tile_request request; request.path = path; - request.stat_only = 0; + request.cur_offset = 0; + request.tile = NULL; + request.tile_expired = 0; + request.tile_mod_time = 0; + request.tile_size = 0; S3_get_object(ctx->ctx, path, NULL, 0, 0, NULL, &getObjectHandler, &request); free(path); if (request.result != S3StatusOK) { - const char *msg = ""; + const char *msg = ""; if (request.error_details && request.error_details->message) { msg = request.error_details->message; } log_message(STORE_LOGLVL_ERR, "store_s3_tile_retrieve: failed to retrieve object: %d(%s)/%s", request.result, S3_get_status_name(request.result), msg); - if (ctx->cache.tile) { - free(ctx->cache.tile); - ctx->cache.tile = NULL; - } - ctx->cache.x = -1; - ctx->cache.y = -1; - ctx->cache.z = -1; return -1; } log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_retrieve: Read object of size %i", request.tile_size); - if (ctx->cache.tile) { - free(ctx->cache.tile); + // extract tile from metatile + + if (request.tile_size < METATILE_HEADER_LEN) { + snprintf(log_msg, PATH_MAX - 1, "Meta file %s too small to contain header\n", path); + free(request.tile); + return -3; } - ctx->cache.tile = request.tile; - ctx->cache.st_stat.size = request.tile_size; - ctx->cache.st_stat.atime = 0; - ctx->cache.st_stat.mtime = request.tile_mod_time; - ctx->cache.st_stat.expired = 0; - ctx->cache.x = x; - ctx->cache.y = y; - ctx->cache.z = z; - strcpy(ctx->cache.xmlname, xmlconfig); - return 1; -} + struct meta_layout *m = (struct meta_layout*) request.tile; -static int store_s3_tile_read(struct storage_backend *store, const char *xmlconfig, const char *options, int x, int y, int z, char *buf, size_t sz, int * compressed, char * log_msg) -{ - struct store_s3_ctx *ctx = (struct store_s3_ctx*) store->storage_ctx; + if (memcmp(m->magic, META_MAGIC, strlen(META_MAGIC))) { + if (memcmp(m->magic, META_MAGIC_COMPRESSED, strlen(META_MAGIC_COMPRESSED))) { + snprintf(log_msg, PATH_MAX - 1, "Meta file %s header magic mismatch\n", path); + free(request.tile); + return -4; + } else { + *compressed = 1; + } + } else { + *compressed = 0; + } - if (store_s3_tile_retrieve(store, xmlconfig, options, x, y, z) <= 0) { - log_message(STORE_LOGLVL_ERR, "store_s3_tile_read: retrieval failed"); - return -1; + if (m->count != (METATILE * METATILE)) { + snprintf(log_msg, PATH_MAX - 1, "Meta file %s header bad count %d != %d\n", path, m->count, METATILE * METATILE); + free(request.tile); + return -5; } - if (ctx->cache.st_stat.size > sz) { - log_message(STORE_LOGLVL_ERR, "store_s3_tile_read: buffer not big enough for tile (%i < %i)", sz, ctx->cache.st_stat.size); - return -1; + + int buffer_offset = m->index[tile_offset].offset; + int tile_size = m->index[tile_offset].size; + + if (tile_size > sz) { + snprintf(log_msg, PATH_MAX - 1, "tile of length %d too big to fit buffer of length %zd\n", tile_size, sz); + free(request.tile); + return -6; } - memcpy(buf, ctx->cache.tile, ctx->cache.st_stat.size); - return ctx->cache.st_stat.size; + + memcpy(buf, request.tile + buffer_offset, tile_size); + + free(request.tile); + request.tile = NULL; + + return tile_size; } static struct stat_info store_s3_tile_stat(struct storage_backend *store, const char *xmlconfig, const char *options, int x, int y, int z) @@ -204,17 +198,11 @@ static struct stat_info store_s3_tile_stat(struct storage_backend *store, const struct store_s3_ctx *ctx = (struct store_s3_ctx*) store->storage_ctx; char *path = NULL; - if ((ctx->cache.x == x) && (ctx->cache.y == y) && (ctx->cache.z == z) - && (strcmp(ctx->cache.xmlname, xmlconfig) == 0)) { - log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_stat: got a cached tile"); - return ctx->cache.st_stat; - } - log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_stat: Fetching tile properties"); path = malloc(PATH_MAX); - store_s3_xyz_to_storagekey(store, x, y, z, path); + store_s3_xyz_to_storagekey(store, xmlconfig, options, x, y, z, path, PATH_MAX); log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_stat: getting properties for object %s", path); struct S3ResponseHandler responseHandler; @@ -223,18 +211,24 @@ static struct stat_info store_s3_tile_stat(struct storage_backend *store, const struct s3_tile_request request; request.path = path; - request.stat_only = 1; + request.error_details = NULL; + request.cur_offset = 0; + request.result = S3StatusOK; + request.tile = NULL; + request.tile_expired = 0; + request.tile_mod_time = 0; + request.tile_size = 0; S3_head_object(ctx->ctx, path, NULL, &responseHandler, &request); free(path); struct stat_info tile_stat; if (request.result != S3StatusOK) { - const char *msg = ""; + const char *msg = ""; if (request.error_details && request.error_details->message) { msg = request.error_details->message; } - log_message(STORE_LOGLVL_ERR, "store_s3_tile_retrieve: failed to retrieve object properties: %d(%s)/%s", request.result, S3_get_status_name(request.result), msg); + log_message(STORE_LOGLVL_ERR, "store_s3_tile_stat: failed to retrieve object properties: %d(%s)/%s", request.result, S3_get_status_name(request.result), msg); tile_stat.size = -1; tile_stat.expired = 0; tile_stat.mtime = 0; @@ -243,7 +237,7 @@ static struct stat_info store_s3_tile_stat(struct storage_backend *store, const return tile_stat; } - log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_retrieve: Read properties"); + log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_stat: Read properties"); tile_stat.size = request.tile_size; tile_stat.expired = request.tile_expired; @@ -253,23 +247,23 @@ static struct stat_info store_s3_tile_stat(struct storage_backend *store, const return tile_stat; } -static char* store_s3_tile_storage_id(struct storage_backend *store, const char *xmlconfig, const char *options, int x, int y, int z, char * string) +static char* store_s3_tile_storage_id(struct storage_backend *store, const char *xmlconfig, const char *options, int x, int y, int z, char *string) { - return store_s3_xyz_to_storagekey(store, x, y, z, string); + // FIXME: assumes PATH_MAX for length of provided string + store_s3_xyz_to_storagekey(store, xmlconfig, options, x, y, z, string, PATH_MAX); + return string; } -static int store_s3_tile_write(struct storage_backend *store, const char *xmlconfig, const char *options, int x, int y, int z, const char *buf, int sz) +static int store_s3_metatile_write(struct storage_backend *store, const char *xmlconfig, const char *options, int x, int y, int z, const char *buf, int sz) { struct store_s3_ctx *ctx = (struct store_s3_ctx*) store->storage_ctx; char *path = malloc(PATH_MAX); - store_s3_xyz_to_storagekey(store, x, y, z, path); - log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_write: storing object %s", path); + store_s3_xyz_to_storagekey(store, xmlconfig, options, x, y, z, path, PATH_MAX); + log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_write: storing object %s, size %ld", path, sz); struct S3PutObjectHandler putObjectHandler; - putObjectHandler.responseHandler.propertiesCallback = - &store_s3_properties_callback; - putObjectHandler.responseHandler.completeCallback = - &store_s3_complete_callback; + putObjectHandler.responseHandler.propertiesCallback = &store_s3_properties_callback; + putObjectHandler.responseHandler.completeCallback = &store_s3_complete_callback; putObjectHandler.putObjectDataCallback = &store_s3_put_object_data_callback; struct s3_tile_request request; @@ -278,28 +272,46 @@ static int store_s3_tile_write(struct storage_backend *store, const char *xmlcon request.tile_size = sz; request.cur_offset = 0; - S3_put_object(ctx->ctx, path, sz, NULL, NULL, &putObjectHandler, &request); + S3PutProperties props; + props.contentType = "application/octet-stream"; + props.cacheControl = NULL; + props.cannedAcl = S3CannedAclPrivate; // results in no ACL header in POST + props.contentDispositionFilename = NULL; + props.contentEncoding = NULL; + props.expires = -1; + props.md5 = NULL; + props.metaData = NULL; + props.metaDataCount = 0; + props.useServerSideEncryption = 0; + + S3_put_object(ctx->ctx, path, sz, &props, NULL, &putObjectHandler, &request); free(path); if (request.result != S3StatusOK) { - const char *msg = ""; - if (request.error_details && request.error_details->message) { - msg = request.error_details->message; + const char *msg = ""; + const char *msg2 = ""; + if (request.error_details) { + if (request.error_details->message) { + msg = request.error_details->message; + } + if (request.error_details->furtherDetails) { + msg2 = request.error_details->furtherDetails; + } } - log_message(STORE_LOGLVL_ERR, "store_s3_tile_write: failed to write object: %d(%s)/%s", request.result, S3_get_status_name(request.result), msg); + log_message(STORE_LOGLVL_ERR, "store_s3_tile_write: failed to write object: %d(%s)/%s%s", request.result, S3_get_status_name(request.result), msg, msg2); return -1; } - log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_write: Wrote object of size %i", request.tile_size); + log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_write: Wrote object of size %i", sz); return sz; } -static int store_s3_tile_delete(struct storage_backend *store, const char *xmlconfig, int x, int y, int z) +static int store_s3_metatile_delete(struct storage_backend *store, const char *xmlconfig, int x, int y, int z) { struct store_s3_ctx *ctx = (struct store_s3_ctx*) store->storage_ctx; char *path = malloc(PATH_MAX); - store_s3_xyz_to_storagekey(store, x, y, z, path); + store_s3_xyz_to_storagekey(store, xmlconfig, NULL, x, y, z, path, PATH_MAX); log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_write: deleting object %s", path); struct S3ResponseHandler responseHandler; @@ -308,12 +320,11 @@ static int store_s3_tile_delete(struct storage_backend *store, const char *xmlco struct s3_tile_request request; request.path = path; - request.stat_only = 1; S3_delete_object(ctx->ctx, path, NULL, &responseHandler, &request); if (request.result != S3StatusOK) { - const char *msg = ""; + const char *msg = ""; if (request.error_details && request.error_details->message) { msg = request.error_details->message; } @@ -326,11 +337,11 @@ static int store_s3_tile_delete(struct storage_backend *store, const char *xmlco return 0; } -static int store_s3_tile_expire(struct storage_backend *store, const char *xmlconfig, int x, int y, int z) +static int store_s3_metatile_expire(struct storage_backend *store, const char *xmlconfig, int x, int y, int z) { struct store_s3_ctx *ctx = (struct store_s3_ctx*) store->storage_ctx; char *path = malloc(PATH_MAX); - store_s3_xyz_to_storagekey(store, x, y, z, path); + store_s3_xyz_to_storagekey(store, xmlconfig, NULL, x, y, z, path, PATH_MAX); log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_expire: expiring object %s", path); struct S3ResponseHandler responseHandler; @@ -345,17 +356,24 @@ static int store_s3_tile_expire(struct storage_backend *store, const char *xmlco expireTag.name = "expired"; expireTag.value = "1"; - struct S3PutProperties putProperties; - putProperties.metaDataCount = 1; - putProperties.metaData = &expireTag; + S3PutProperties props; + props.contentType = "application/octet-stream"; + props.cacheControl = NULL; + props.cannedAcl = S3CannedAclPrivate; // results in no ACL header in POST + props.contentDispositionFilename = NULL; + props.contentEncoding = NULL; + props.expires = -1; + props.md5 = NULL; + props.metaDataCount = 1; + props.metaData = &expireTag; int64_t lastModified; - S3_copy_object(ctx->ctx, path, ctx->ctx->bucketName, path, &putProperties, &lastModified, 0, NULL, NULL, &responseHandler, &request); + S3_copy_object(ctx->ctx, path, ctx->ctx->bucketName, path, &props, &lastModified, 0, NULL, NULL, &responseHandler, &request); free(path); if (request.result != S3StatusOK) { - const char *msg = ""; + const char *msg = ""; if (request.error_details && request.error_details->message) { msg = request.error_details->message; } @@ -370,19 +388,21 @@ static int store_s3_tile_expire(struct storage_backend *store, const char *xmlco static int store_s3_close_storage(struct storage_backend *store) { - struct store_s3_ctx * ctx = (struct store_s3_ctx *) (store->storage_ctx); + struct store_s3_ctx *ctx = (struct store_s3_ctx*) store->storage_ctx; - if (ctx->cache.tile) - free(ctx->cache.tile); S3_deinitialize(); free(ctx); free(store); + store_s3_initialized = 0; return 0; } static char* url_decode(const char *src) { + if (NULL == src) { + return NULL; + } char *dst = (char*) malloc(strlen(src) + 1); dst[0] = '\0'; while (*src) { @@ -426,15 +446,9 @@ struct storage_backend* init_storage_s3(const char *connection_string) return NULL; } - ctx->cache.x = -1; - ctx->cache.y = -1; - ctx->cache.z = -1; - ctx->cache.tile = NULL; - ctx->cache.xmlname[0] = 0; - pthread_mutex_lock(&qLock); if (!store_s3_initialized) { - log_message(STORE_LOGLVL_DEBUG, "init_storage_s3: Global init of libs3", connection_string); + log_message(STORE_LOGLVL_DEBUG, "init_storage_s3: global init of libs3", connection_string); res = S3_initialize(NULL, S3_INIT_ALL, NULL); store_s3_initialized = 1; } else { @@ -452,18 +466,30 @@ struct storage_backend* init_storage_s3(const char *connection_string) ctx->ctx = malloc(sizeof(struct S3BucketContext)); char *fullurl = strdup(connection_string); + // advance past "s3://" fullurl = &fullurl[5]; - ctx->ctx->accessKeyId = url_decode(strsep(&fullurl, ":")); + ctx->ctx->accessKeyId = strsep(&fullurl, ":"); if (strchr(fullurl, '@')) { - ctx->ctx->secretAccessKey = url_decode(strsep(&fullurl, "@")); - ctx->ctx->hostName = url_decode(strsep(&fullurl, "/")); + ctx->ctx->secretAccessKey = strsep(&fullurl, "@"); + ctx->ctx->hostName = strsep(&fullurl, "/"); } else { - ctx->ctx->secretAccessKey = url_decode(strsep(&fullurl, "/")); + ctx->ctx->secretAccessKey = strsep(&fullurl, "/"); + ctx->ctx->hostName = NULL; } - ctx->ctx->bucketName = url_decode(strsep(&fullurl, "/")); + ctx->ctx->bucketName = strsep(&fullurl, "/"); + + ctx->basepath = fullurl; + + ctx->ctx->accessKeyId = url_decode(ctx->ctx->accessKeyId); + ctx->ctx->secretAccessKey = url_decode(ctx->ctx->secretAccessKey); + ctx->ctx->hostName = url_decode(ctx->ctx->hostName); + ctx->ctx->bucketName = url_decode(ctx->ctx->bucketName); + ctx->ctx->protocol = S3ProtocolHTTPS; + ctx->ctx->securityToken = NULL; + ctx->ctx->uriStyle = S3UriStyleVirtualHost; - ctx->basepath = url_decode(fullurl); + ctx->basepath = url_decode(ctx->basepath); log_message(STORE_LOGLVL_DEBUG, "init_storage_s3 completed keyid: %s, key: %s, bucket: %s, basepath: %s", ctx->ctx->accessKeyId, ctx->ctx->secretAccessKey, ctx->ctx->bucketName, ctx->basepath); @@ -471,9 +497,9 @@ struct storage_backend* init_storage_s3(const char *connection_string) store->tile_read = &store_s3_tile_read; store->tile_stat = &store_s3_tile_stat; - store->metatile_write = &store_s3_tile_write; - store->metatile_delete = &store_s3_tile_delete; - store->metatile_expire = &store_s3_tile_expire; + store->metatile_write = &store_s3_metatile_write; + store->metatile_delete = &store_s3_metatile_delete; + store->metatile_expire = &store_s3_metatile_expire; store->tile_storage_id = &store_s3_tile_storage_id; store->close_storage = &store_s3_close_storage; From 11fd2bc661f3db529e1086628d4d092fa3b74918 Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Fri, 15 Jan 2016 14:01:22 -0700 Subject: [PATCH 09/35] add environment variable support to S3 credential configuration --- src/store_s3.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/store_s3.c b/src/store_s3.c index bc8a4d2f..966957c8 100644 --- a/src/store_s3.c +++ b/src/store_s3.c @@ -481,7 +481,19 @@ struct storage_backend* init_storage_s3(const char *connection_string) ctx->basepath = fullurl; + if (strstr(ctx->ctx->accessKeyId, "${") == ctx->ctx->accessKeyId && strrchr(ctx->ctx->accessKeyId, '}') == (ctx->ctx->accessKeyId + strlen(ctx->ctx->accessKeyId) - 1)) { + char tmp[strlen(ctx->ctx->accessKeyId) + 1]; + strcpy(tmp, ctx->ctx->accessKeyId); + tmp[strlen(tmp) - 1] = '\0'; + ctx->ctx->accessKeyId = getenv(tmp + 2); + } ctx->ctx->accessKeyId = url_decode(ctx->ctx->accessKeyId); + if (strstr(ctx->ctx->secretAccessKey, "${") == ctx->ctx->secretAccessKey && strrchr(ctx->ctx->secretAccessKey, '}') == (ctx->ctx->secretAccessKey + strlen(ctx->ctx->secretAccessKey) - 1)) { + char tmp[strlen(ctx->ctx->secretAccessKey) + 1]; + strcpy(tmp, ctx->ctx->secretAccessKey); + tmp[strlen(tmp) - 1] = '\0'; + ctx->ctx->secretAccessKey = getenv(tmp + 2); + } ctx->ctx->secretAccessKey = url_decode(ctx->ctx->secretAccessKey); ctx->ctx->hostName = url_decode(ctx->ctx->hostName); ctx->ctx->bucketName = url_decode(ctx->ctx->bucketName); From ff272be04e73d941cdf5c54ea191ceeb8c6829eb Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Fri, 15 Jan 2016 15:05:55 -0700 Subject: [PATCH 10/35] don't chdir when forking - this allows systemd to configure the working directory --- src/daemon.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/daemon.c b/src/daemon.c index 56a5b0b4..d1f864af 100644 --- a/src/daemon.c +++ b/src/daemon.c @@ -932,7 +932,7 @@ int main(int argc, char **argv) if (foreground) { fprintf(stderr, "Running in foreground mode...\n"); } else { - if (daemon(0, 0) != 0) { + if (daemon(1, 0) != 0) { fprintf(stderr, "can't daemonize: %s\n", strerror(errno)); } /* write pid file */ From 4c851bff15938248a766bd5f640ea75e8878cd56 Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Fri, 15 Jan 2016 15:32:18 -0700 Subject: [PATCH 11/35] let systemd handle stdout/stderr too --- src/daemon.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/daemon.c b/src/daemon.c index d1f864af..f84c5968 100644 --- a/src/daemon.c +++ b/src/daemon.c @@ -932,7 +932,7 @@ int main(int argc, char **argv) if (foreground) { fprintf(stderr, "Running in foreground mode...\n"); } else { - if (daemon(1, 0) != 0) { + if (daemon(1, 1) != 0) { fprintf(stderr, "can't daemonize: %s\n", strerror(errno)); } /* write pid file */ From fa298941d1dcec6d18cfc1817bcc234773bb893a Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Fri, 15 Jan 2016 16:22:00 -0700 Subject: [PATCH 12/35] clean up some uninitialized values --- src/store_s3.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/store_s3.c b/src/store_s3.c index 966957c8..0a157195 100644 --- a/src/store_s3.c +++ b/src/store_s3.c @@ -271,6 +271,9 @@ static int store_s3_metatile_write(struct storage_backend *store, const char *xm request.tile = (char*) buf; request.tile_size = sz; request.cur_offset = 0; + request.tile_expired = 0; + request.result = S3StatusOK; + request.error_details = NULL; S3PutProperties props; props.contentType = "application/octet-stream"; From 485cce0a3626f1f39701bc61bde11594ca047e75 Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Fri, 15 Jan 2016 16:46:35 -0700 Subject: [PATCH 13/35] fix more uninitialized values to make valgrind happy --- src/gen_tile_test.cpp | 4 ++++ src/store_s3.c | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/gen_tile_test.cpp b/src/gen_tile_test.cpp index f306746e..188d4a07 100644 --- a/src/gen_tile_test.cpp +++ b/src/gen_tile_test.cpp @@ -1238,6 +1238,10 @@ TEST_CASE("storage-backend/s3", "S3 tile storage backend") { ctx.secretAccessKey = accesskey; ctx.bucketName = bucketname; ctx.hostName = NULL; + ctx.protocol = S3ProtocolHTTPS; + ctx.uriStyle = S3UriStyleVirtualHost; + ctx.securityToken = NULL; + S3ResponseHandler handler; handler.completeCallback = &test_s3_complete_callback; handler.propertiesCallback = &test_s3_properties_callback; diff --git a/src/store_s3.c b/src/store_s3.c index 0a157195..c49bef3b 100644 --- a/src/store_s3.c +++ b/src/store_s3.c @@ -323,8 +323,16 @@ static int store_s3_metatile_delete(struct storage_backend *store, const char *x struct s3_tile_request request; request.path = path; + request.error_details = NULL; + request.cur_offset = 0; + request.result = S3StatusOK; + request.tile = NULL; + request.tile_expired = 0; + request.tile_mod_time = 0; + request.tile_size = 0; S3_delete_object(ctx->ctx, path, NULL, &responseHandler, &request); + free(path); if (request.result != S3StatusOK) { const char *msg = ""; @@ -353,7 +361,13 @@ static int store_s3_metatile_expire(struct storage_backend *store, const char *x struct s3_tile_request request; request.path = path; + request.error_details = NULL; request.cur_offset = 0; + request.result = S3StatusOK; + request.tile = NULL; + request.tile_expired = 0; + request.tile_mod_time = 0; + request.tile_size = 0; struct S3NameValue expireTag; expireTag.name = "expired"; @@ -488,14 +502,28 @@ struct storage_backend* init_storage_s3(const char *connection_string) char tmp[strlen(ctx->ctx->accessKeyId) + 1]; strcpy(tmp, ctx->ctx->accessKeyId); tmp[strlen(tmp) - 1] = '\0'; - ctx->ctx->accessKeyId = getenv(tmp + 2); + char *val = getenv(tmp + 2); + if (NULL == val) { + log_message(STORE_LOGLVL_ERR, "init_storage_s3: environment variable %s not defined when initializing S3 configuration!", tmp + 2); + free(ctx); + free(store); + return NULL; + } + ctx->ctx->accessKeyId = val; } ctx->ctx->accessKeyId = url_decode(ctx->ctx->accessKeyId); if (strstr(ctx->ctx->secretAccessKey, "${") == ctx->ctx->secretAccessKey && strrchr(ctx->ctx->secretAccessKey, '}') == (ctx->ctx->secretAccessKey + strlen(ctx->ctx->secretAccessKey) - 1)) { char tmp[strlen(ctx->ctx->secretAccessKey) + 1]; strcpy(tmp, ctx->ctx->secretAccessKey); tmp[strlen(tmp) - 1] = '\0'; - ctx->ctx->secretAccessKey = getenv(tmp + 2); + char *val = getenv(tmp + 2); + if (NULL == val) { + log_message(STORE_LOGLVL_ERR, "init_storage_s3: environment variable %s not defined when initializing S3 configuration!", tmp + 2); + free(ctx); + free(store); + return NULL; + } + ctx->ctx->secretAccessKey = val; } ctx->ctx->secretAccessKey = url_decode(ctx->ctx->secretAccessKey); ctx->ctx->hostName = url_decode(ctx->ctx->hostName); From ab5829ae91e1524a9f171a2c877402e004e464b7 Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Fri, 15 Jan 2016 17:22:35 -0700 Subject: [PATCH 14/35] fix buffer write quantity computation --- src/store_s3.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/store_s3.c b/src/store_s3.c index c49bef3b..0efc4aec 100644 --- a/src/store_s3.c +++ b/src/store_s3.c @@ -94,7 +94,7 @@ int store_s3_put_object_data_callback(int bufferSize, char *buffer, void *callba log_message(STORE_LOGLVL_DEBUG, "store_s3_put_object_data_callback: completed put"); return 0; } - size_t bytesToWrite = MAX(bufferSize, rqst->tile_size - rqst->cur_offset); + size_t bytesToWrite = MIN(bufferSize, rqst->tile_size - rqst->cur_offset); log_message(STORE_LOGLVL_DEBUG, "store_s3_put_object_data_callback: uploading data, writing %ld bytes to buffer, cur offset %ld, new offset %ld", bytesToWrite, rqst->cur_offset, rqst->cur_offset + bytesToWrite); memcpy(buffer, rqst->tile + rqst->cur_offset, bytesToWrite); rqst->cur_offset += bytesToWrite; From 7983efea73f1ad0ad763b025967b5b261770b413 Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Fri, 15 Jan 2016 22:59:46 -0700 Subject: [PATCH 15/35] removed some debug to help performance --- src/store_s3.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/store_s3.c b/src/store_s3.c index 0efc4aec..b891066c 100644 --- a/src/store_s3.c +++ b/src/store_s3.c @@ -76,11 +76,11 @@ S3Status store_s3_object_data_callback(int bufferSize, const char *buffer, void struct s3_tile_request *rqst = (struct s3_tile_request*) callbackData; if (rqst->cur_offset == 0 && rqst->tile == NULL) { - log_message(STORE_LOGLVL_DEBUG, "store_s3_object_data_callback: allocating %ld byte buffer for tile", rqst->tile_size); + //log_message(STORE_LOGLVL_DEBUG, "store_s3_object_data_callback: allocating %ld byte buffer for tile", rqst->tile_size); rqst->tile = malloc(rqst->tile_size); } - log_message(STORE_LOGLVL_DEBUG, "store_s3_object_data_callback: appending %ld bytes to buffer, new offset %ld", bufferSize, rqst->cur_offset + bufferSize); + //log_message(STORE_LOGLVL_DEBUG, "store_s3_object_data_callback: appending %ld bytes to buffer, new offset %ld", bufferSize, rqst->cur_offset + bufferSize); memcpy(rqst->tile + rqst->cur_offset, buffer, bufferSize); rqst->cur_offset += bufferSize; return S3StatusOK; @@ -95,7 +95,7 @@ int store_s3_put_object_data_callback(int bufferSize, char *buffer, void *callba return 0; } size_t bytesToWrite = MIN(bufferSize, rqst->tile_size - rqst->cur_offset); - log_message(STORE_LOGLVL_DEBUG, "store_s3_put_object_data_callback: uploading data, writing %ld bytes to buffer, cur offset %ld, new offset %ld", bytesToWrite, rqst->cur_offset, rqst->cur_offset + bytesToWrite); + //log_message(STORE_LOGLVL_DEBUG, "store_s3_put_object_data_callback: uploading data, writing %ld bytes to buffer, cur offset %ld, new offset %ld", bytesToWrite, rqst->cur_offset, rqst->cur_offset + bytesToWrite); memcpy(buffer, rqst->tile + rqst->cur_offset, bytesToWrite); rqst->cur_offset += bytesToWrite; return bytesToWrite; From 2adc334dc9f71a5d820cc6c292cac572a4984086 Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Mon, 18 Jan 2016 08:08:14 -0700 Subject: [PATCH 16/35] cleanup of logging --- src/store_s3.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/store_s3.c b/src/store_s3.c index b891066c..b5743c53 100644 --- a/src/store_s3.c +++ b/src/store_s3.c @@ -117,10 +117,10 @@ static int store_s3_tile_read(struct storage_backend *store, const char *xmlconf struct store_s3_ctx *ctx = (struct store_s3_ctx*) store->storage_ctx; char *path = malloc(PATH_MAX); - log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_retrieve: fetching tile"); + log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_read: fetching tile"); int tile_offset = store_s3_xyz_to_storagekey(store, xmlconfig, options, x, y, z, path, PATH_MAX); - log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_retrieve: retrieving object %s", path); + log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_read: retrieving object %s", path); struct S3GetObjectHandler getObjectHandler; getObjectHandler.responseHandler.propertiesCallback = &store_s3_properties_callback; @@ -143,11 +143,11 @@ static int store_s3_tile_read(struct storage_backend *store, const char *xmlconf if (request.error_details && request.error_details->message) { msg = request.error_details->message; } - log_message(STORE_LOGLVL_ERR, "store_s3_tile_retrieve: failed to retrieve object: %d(%s)/%s", request.result, S3_get_status_name(request.result), msg); + log_message(STORE_LOGLVL_ERR, "store_s3_tile_read: failed to retrieve object: %d(%s)/%s", request.result, S3_get_status_name(request.result), msg); return -1; } - log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_retrieve: Read object of size %i", request.tile_size); + log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_read: Read object of size %i", request.tile_size); // extract tile from metatile @@ -259,7 +259,7 @@ static int store_s3_metatile_write(struct storage_backend *store, const char *xm struct store_s3_ctx *ctx = (struct store_s3_ctx*) store->storage_ctx; char *path = malloc(PATH_MAX); store_s3_xyz_to_storagekey(store, xmlconfig, options, x, y, z, path, PATH_MAX); - log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_write: storing object %s, size %ld", path, sz); + log_message(STORE_LOGLVL_DEBUG, "store_s3_metatile_write: storing object %s, size %ld", path, sz); struct S3PutObjectHandler putObjectHandler; putObjectHandler.responseHandler.propertiesCallback = &store_s3_properties_callback; @@ -301,11 +301,11 @@ static int store_s3_metatile_write(struct storage_backend *store, const char *xm msg2 = request.error_details->furtherDetails; } } - log_message(STORE_LOGLVL_ERR, "store_s3_tile_write: failed to write object: %d(%s)/%s%s", request.result, S3_get_status_name(request.result), msg, msg2); + log_message(STORE_LOGLVL_ERR, "store_s3_metatile_write: failed to write object: %d(%s)/%s%s", request.result, S3_get_status_name(request.result), msg, msg2); return -1; } - log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_write: Wrote object of size %i", sz); + log_message(STORE_LOGLVL_DEBUG, "store_s3_metatile_write: Wrote object of size %i", sz); return sz; } @@ -315,7 +315,7 @@ static int store_s3_metatile_delete(struct storage_backend *store, const char *x struct store_s3_ctx *ctx = (struct store_s3_ctx*) store->storage_ctx; char *path = malloc(PATH_MAX); store_s3_xyz_to_storagekey(store, xmlconfig, NULL, x, y, z, path, PATH_MAX); - log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_write: deleting object %s", path); + log_message(STORE_LOGLVL_DEBUG, "store_s3_metatile_delete: deleting object %s", path); struct S3ResponseHandler responseHandler; responseHandler.propertiesCallback = &store_s3_properties_callback; @@ -339,11 +339,11 @@ static int store_s3_metatile_delete(struct storage_backend *store, const char *x if (request.error_details && request.error_details->message) { msg = request.error_details->message; } - log_message(STORE_LOGLVL_ERR, "store_s3_tile_delete: failed to delete object: %d(%s)/%s", request.result, S3_get_status_name(request.result), msg); + log_message(STORE_LOGLVL_ERR, "store_s3_metatile_delete: failed to delete object: %d(%s)/%s", request.result, S3_get_status_name(request.result), msg); return -1; } - log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_delete: deleted object"); + log_message(STORE_LOGLVL_DEBUG, "store_s3_metatile_delete: deleted object"); return 0; } @@ -353,7 +353,7 @@ static int store_s3_metatile_expire(struct storage_backend *store, const char *x struct store_s3_ctx *ctx = (struct store_s3_ctx*) store->storage_ctx; char *path = malloc(PATH_MAX); store_s3_xyz_to_storagekey(store, xmlconfig, NULL, x, y, z, path, PATH_MAX); - log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_expire: expiring object %s", path); + log_message(STORE_LOGLVL_DEBUG, "store_s3_metatile_expire: expiring object %s", path); struct S3ResponseHandler responseHandler; responseHandler.propertiesCallback = &store_s3_properties_callback; @@ -394,11 +394,11 @@ static int store_s3_metatile_expire(struct storage_backend *store, const char *x if (request.error_details && request.error_details->message) { msg = request.error_details->message; } - log_message(STORE_LOGLVL_ERR, "store_s3_tile_expire: failed to update object: %d(%s)/%s", request.result, S3_get_status_name(request.result), msg); + log_message(STORE_LOGLVL_ERR, "store_s3_metatile_expire: failed to update object: %d(%s)/%s", request.result, S3_get_status_name(request.result), msg); return -1; } - log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_expire: Updated object metadata"); + log_message(STORE_LOGLVL_DEBUG, "store_s3_metatile_expire: Updated object metadata"); return 0; } From e6eff63a6a90a3ffb540b86888f422fb80edd812 Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Tue, 19 Jan 2016 10:28:06 -0700 Subject: [PATCH 17/35] added S3 reference to README --- readme.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readme.txt b/readme.txt index 0bcf0958..76c5ce6f 100644 --- a/readme.txt +++ b/readme.txt @@ -46,8 +46,8 @@ daemon to render (or re-render) the tile. resources on the server and how out of date they are. 4) Use tile storage other than a plain posix file system. -e.g it can store tiles in a ceph object store, or proxy them -from another tile server. +e.g it can store tiles in a ceph object store, an Amazon S3 bucket, +or proxy them from another tile server. 5) Tile expiry. It estimates when the tile is next likely to be rendered and adds the appropriate HTTP From 964a85d3bdbd81930b3e0175e3549d3e9a3cf00c Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Thu, 21 Jan 2016 16:42:23 -0700 Subject: [PATCH 18/35] improved autoconf support for libs3 - now allows library location to be specified to configure --- Makefile.am | 5 +++-- configure.ac | 45 ++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/Makefile.am b/Makefile.am index a836930b..595d925d 100644 --- a/Makefile.am +++ b/Makefile.am @@ -6,10 +6,11 @@ endif ACLOCAL_AMFLAGS = -I m4 AM_CPPFLAGS = $(PTHREAD_CFLAGS) -DSYSTEM_LIBINIPARSER=@SYSTEM_LIBINIPARSER@ +AM_CFLAGS = $(STORE_CFLAGS) STORE_SOURCES = src/store.c src/store_file.c src/store_file_utils.c src/store_memcached.c src/store_rados.c src/store_ro_http_proxy.c src/store_ro_composite.c src/store_null.c src/store_s3.c STORE_LDFLAGS = $(LIBMEMCACHED_LDFLAGS) $(LIBRADOS_LDFLAGS) $(LIBCURL) $(LIBS3_LDFLAGS) -STORE_CPPFLAGS = +STORE_CFLAGS = $(LIBS3_CFLAGS) bin_PROGRAMS = renderd render_expired render_list render_speedtest render_old noinst_PROGRAMS = gen_tile_test @@ -34,7 +35,7 @@ render_old_SOURCES = src/store_file_utils.c src/render_old.c src/sys_utils.c src render_old_LDADD = $(PTHREAD_CFLAGS) #convert_meta_SOURCES = src/dir_utils.c src/store.c src/convert_meta.c gen_tile_test_SOURCES = src/gen_tile_test.cpp src/metatile.cpp src/request_queue.c src/protocol_helper.c src/daemon.c src/daemon_compat.c src/gen_tile.cpp src/sys_utils.c src/cache_expire.c src/parameterize_style.cpp $(STORE_SOURCES) -gen_tile_test_CFLAGS = -DMAIN_ALREADY_DEFINED $(PTHREAD_CFLAGS) +gen_tile_test_CFLAGS = -DMAIN_ALREADY_DEFINED $(PTHREAD_CFLAGS) $(STORE_CFLAGS) gen_tile_test_CXXFLAGS = $(MAPNIK_CFLAGS) gen_tile_test_LDADD = $(PTHREAD_CFLAGS) $(MAPNIK_LDFLAGS) $(STORE_LDFLAGS) -liniparser if !SYSTEM_LIBINIPARSER diff --git a/configure.ac b/configure.ac index 1965f377..a951061d 100644 --- a/configure.ac +++ b/configure.ac @@ -53,11 +53,46 @@ AC_CHECK_LIB(rados, rados_version, [ LIBRADOS_LDFLAGS='-lrados' AC_SUBST(LIBRADOS_LDFLAGS) ][]) -AC_CHECK_LIB(s3, S3_deinitialize, [ - AC_DEFINE([HAVE_LIBS3], [1], [Have found libs3]) - LIBS3_LDFLAGS='-ls3' - AC_SUBST(LIBS3_LDFLAGS) -][]) + +AC_ARG_WITH([libs3], + [AS_HELP_STRING([--with-libs3[=DIR]],[path to libs3])], + [ + libs3_dir="$withval" + if test "$libs3_dir" != "no"; then + if test "$libs3_dir" != "yes"; then + AC_MSG_CHECKING([for libs3]) + if test -f "$libs3_dir/include/libs3.h"; then + AC_DEFINE([HAVE_LIBS3], [1], [Have found libs3]) + LIBS3_LDFLAGS='-ls3' + AC_SUBST(LIBS3_LDFLAGS) + LIBS3_CFLAGS="-I$libs3_dir/include" + AC_SUBST(LIBS3_CFLAGS) + AC_MSG_RESULT($libs3_dir) + else + AC_MSG_ERROR([no libs3 found at $libs3_dir]) + fi + else + AC_CHECK_LIB(s3, S3_deinitialize, + [ + AC_DEFINE([HAVE_LIBS3], [1], [Have found libs3]) + LIBS3_LDFLAGS='-ls3' + AC_SUBST(LIBS3_LDFLAGS) + ], + [ + AC_MSG_ERROR([no libs3 found]) + ]) + fi + fi + ], + [ + AC_CHECK_LIB(s3, S3_deinitialize, + [ + AC_DEFINE([HAVE_LIBS3], [1], [Have found libs3]) + LIBS3_LDFLAGS='-ls3' + AC_SUBST(LIBS3_LDFLAGS) + ], + []) + ]) AC_CHECK_FUNCS([bzero gethostbyname gettimeofday inet_ntoa memset mkdir pow select socket strchr strdup strerror strrchr strstr strtol strtoul utime],[],[AC_MSG_ERROR([One of the required functions was not found])]) AC_CHECK_FUNCS([daemon getloadavg],[],[]) From 5007a75453c9f0954c5f4d20014c8b536bc3452d Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Fri, 22 Jan 2016 11:33:58 -0700 Subject: [PATCH 19/35] fix library search path when specifying an alternate location for libs3 library --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index a951061d..62964bad 100644 --- a/configure.ac +++ b/configure.ac @@ -63,7 +63,7 @@ AC_ARG_WITH([libs3], AC_MSG_CHECKING([for libs3]) if test -f "$libs3_dir/include/libs3.h"; then AC_DEFINE([HAVE_LIBS3], [1], [Have found libs3]) - LIBS3_LDFLAGS='-ls3' + LIBS3_LDFLAGS="-L$libs3_dir/lib -ls3" AC_SUBST(LIBS3_LDFLAGS) LIBS3_CFLAGS="-I$libs3_dir/include" AC_SUBST(LIBS3_CFLAGS) From 442e0faa546999c7f58fa03e67387e4fd7d89a33 Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Mon, 25 Jan 2016 12:27:48 -0700 Subject: [PATCH 20/35] add environment variable support to bucket name and basepath --- src/store_s3.c | 73 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 51 insertions(+), 22 deletions(-) diff --git a/src/store_s3.c b/src/store_s3.c index b5743c53..8d66428e 100644 --- a/src/store_s3.c +++ b/src/store_s3.c @@ -480,27 +480,27 @@ struct storage_backend* init_storage_s3(const char *connection_string) } // parse out the context information from the URL: s3://:[@]/[/] - ctx->ctx = malloc(sizeof(struct S3BucketContext)); + struct S3BucketContext *bctx = ctx->ctx = malloc(sizeof(struct S3BucketContext)); char *fullurl = strdup(connection_string); // advance past "s3://" fullurl = &fullurl[5]; - ctx->ctx->accessKeyId = strsep(&fullurl, ":"); + bctx->accessKeyId = strsep(&fullurl, ":"); if (strchr(fullurl, '@')) { - ctx->ctx->secretAccessKey = strsep(&fullurl, "@"); - ctx->ctx->hostName = strsep(&fullurl, "/"); + bctx->secretAccessKey = strsep(&fullurl, "@"); + bctx->hostName = strsep(&fullurl, "/"); } else { - ctx->ctx->secretAccessKey = strsep(&fullurl, "/"); - ctx->ctx->hostName = NULL; + bctx->secretAccessKey = strsep(&fullurl, "/"); + bctx->hostName = NULL; } - ctx->ctx->bucketName = strsep(&fullurl, "/"); + bctx->bucketName = strsep(&fullurl, "/"); ctx->basepath = fullurl; - if (strstr(ctx->ctx->accessKeyId, "${") == ctx->ctx->accessKeyId && strrchr(ctx->ctx->accessKeyId, '}') == (ctx->ctx->accessKeyId + strlen(ctx->ctx->accessKeyId) - 1)) { - char tmp[strlen(ctx->ctx->accessKeyId) + 1]; - strcpy(tmp, ctx->ctx->accessKeyId); + if (strstr(bctx->accessKeyId, "${") == bctx->accessKeyId && strrchr(bctx->accessKeyId, '}') == (bctx->accessKeyId + strlen(bctx->accessKeyId) - 1)) { + char tmp[strlen(bctx->accessKeyId) + 1]; + strcpy(tmp, bctx->accessKeyId); tmp[strlen(tmp) - 1] = '\0'; char *val = getenv(tmp + 2); if (NULL == val) { @@ -509,12 +509,29 @@ struct storage_backend* init_storage_s3(const char *connection_string) free(store); return NULL; } - ctx->ctx->accessKeyId = val; + bctx->accessKeyId = val; } - ctx->ctx->accessKeyId = url_decode(ctx->ctx->accessKeyId); - if (strstr(ctx->ctx->secretAccessKey, "${") == ctx->ctx->secretAccessKey && strrchr(ctx->ctx->secretAccessKey, '}') == (ctx->ctx->secretAccessKey + strlen(ctx->ctx->secretAccessKey) - 1)) { - char tmp[strlen(ctx->ctx->secretAccessKey) + 1]; - strcpy(tmp, ctx->ctx->secretAccessKey); + bctx->accessKeyId = url_decode(bctx->accessKeyId); + + if (strstr(bctx->secretAccessKey, "${") == bctx->secretAccessKey && strrchr(bctx->secretAccessKey, '}') == (bctx->secretAccessKey + strlen(bctx->secretAccessKey) - 1)) { + char tmp[strlen(bctx->secretAccessKey) + 1]; + strcpy(tmp, bctx->secretAccessKey); + tmp[strlen(tmp) - 1] = '\0'; + char *val = getenv(tmp + 2); + if (NULL == val) { + log_message(STORE_LOGLVL_ERR, "init_storage_s3: environment variable %s not defined when initializing S3 configuration!", tmp + 2); + free(ctx); + free(store); + return NULL; + } + bctx->secretAccessKey = val; + } + bctx->secretAccessKey = url_decode(bctx->secretAccessKey); + + bctx->hostName = url_decode(bctx->hostName); + if (strstr(bctx->bucketName, "${") == bctx->bucketName && strrchr(bctx->bucketName, '}') == (bctx->bucketName + strlen(bctx->bucketName) - 1)) { + char tmp[strlen(bctx->bucketName) + 1]; + strcpy(tmp, bctx->bucketName); tmp[strlen(tmp) - 1] = '\0'; char *val = getenv(tmp + 2); if (NULL == val) { @@ -523,15 +540,27 @@ struct storage_backend* init_storage_s3(const char *connection_string) free(store); return NULL; } - ctx->ctx->secretAccessKey = val; + bctx->bucketName = val; } - ctx->ctx->secretAccessKey = url_decode(ctx->ctx->secretAccessKey); - ctx->ctx->hostName = url_decode(ctx->ctx->hostName); - ctx->ctx->bucketName = url_decode(ctx->ctx->bucketName); - ctx->ctx->protocol = S3ProtocolHTTPS; - ctx->ctx->securityToken = NULL; - ctx->ctx->uriStyle = S3UriStyleVirtualHost; + bctx->bucketName = url_decode(bctx->bucketName); + bctx->protocol = S3ProtocolHTTPS; + bctx->securityToken = NULL; + bctx->uriStyle = S3UriStyleVirtualHost; + + if (strstr(ctx->basepath, "${") == ctx->basepath && strrchr(ctx->basepath, '}') == (ctx->basepath + strlen(ctx->basepath) - 1)) { + char tmp[strlen(ctx->basepath) + 1]; + strcpy(tmp, ctx->basepath); + tmp[strlen(tmp) - 1] = '\0'; + char *val = getenv(tmp + 2); + if (NULL == val) { + log_message(STORE_LOGLVL_ERR, "init_storage_s3: environment variable %s not defined when initializing S3 configuration!", tmp + 2); + free(ctx); + free(store); + return NULL; + } + ctx->basepath = val; + } ctx->basepath = url_decode(ctx->basepath); log_message(STORE_LOGLVL_DEBUG, "init_storage_s3 completed keyid: %s, key: %s, bucket: %s, basepath: %s", ctx->ctx->accessKeyId, ctx->ctx->secretAccessKey, ctx->ctx->bucketName, ctx->basepath); From 42ed8bd39587872314d497b82a688445add738c7 Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Mon, 25 Jan 2016 15:23:12 -0700 Subject: [PATCH 21/35] improve error logging upon bind failure --- src/daemon.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/daemon.c b/src/daemon.c index f84c5968..28737d83 100644 --- a/src/daemon.c +++ b/src/daemon.c @@ -430,8 +430,8 @@ int server_socket_init(renderd_config *sConfig) { addrI.sin6_addr = in6addr_any; addrI.sin6_port = htons(sConfig->ipport); if (bind(fd, (struct sockaddr *) &addrI, sizeof(addrI)) < 0) { - fprintf(stderr, "socket bind failed for: %s:%i\n", - sConfig->iphostname, sConfig->ipport); + fprintf(stderr, "socket bind failed for: %s:%i: %s\n", + sConfig->iphostname, sConfig->ipport, strerror(errno)); close(fd); exit(3); } @@ -453,7 +453,8 @@ int server_socket_init(renderd_config *sConfig) { old = umask(0); // Need daemon socket to be writeable by apache if (bind(fd, (struct sockaddr *) &addrU, sizeof(addrU)) < 0) { - fprintf(stderr, "socket bind failed for: %s\n", sConfig->socketname); + fprintf(stderr, "socket bind failed for: %s: %s\n", + sConfig->socketname, strerror(errno)); close(fd); exit(3); } From 60783e1f41bd798f573e0c24307cd44b9e874f3e Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Tue, 26 Jan 2016 10:12:28 -0700 Subject: [PATCH 22/35] fix compilation error when specifying non-standard libs3 directory --- Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.am b/Makefile.am index 595d925d..0d235544 100644 --- a/Makefile.am +++ b/Makefile.am @@ -36,7 +36,7 @@ render_old_LDADD = $(PTHREAD_CFLAGS) #convert_meta_SOURCES = src/dir_utils.c src/store.c src/convert_meta.c gen_tile_test_SOURCES = src/gen_tile_test.cpp src/metatile.cpp src/request_queue.c src/protocol_helper.c src/daemon.c src/daemon_compat.c src/gen_tile.cpp src/sys_utils.c src/cache_expire.c src/parameterize_style.cpp $(STORE_SOURCES) gen_tile_test_CFLAGS = -DMAIN_ALREADY_DEFINED $(PTHREAD_CFLAGS) $(STORE_CFLAGS) -gen_tile_test_CXXFLAGS = $(MAPNIK_CFLAGS) +gen_tile_test_CXXFLAGS = $(MAPNIK_CFLAGS) $(STORE_CFLAGS) gen_tile_test_LDADD = $(PTHREAD_CFLAGS) $(MAPNIK_LDFLAGS) $(STORE_LDFLAGS) -liniparser if !SYSTEM_LIBINIPARSER gen_tile_test_SOURCES += iniparser3.0b/libiniparser.la From 5538f63565be815a5e2ee2da3d786f30b932b912 Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Wed, 3 Feb 2016 19:17:32 -0700 Subject: [PATCH 23/35] refined debug logging --- src/store_s3.c | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/store_s3.c b/src/store_s3.c index 8d66428e..74ac2c67 100644 --- a/src/store_s3.c +++ b/src/store_s3.c @@ -104,10 +104,10 @@ int store_s3_put_object_data_callback(int bufferSize, char *buffer, void *callba void store_s3_complete_callback(S3Status status, const S3ErrorDetails *errorDetails, void *callbackData) { struct s3_tile_request *rqst = (struct s3_tile_request*) callbackData; - log_message(STORE_LOGLVL_DEBUG, "store_s3_complete_callback: request complete, status %d (%s)", status, S3_get_status_name(status)); - if (errorDetails && errorDetails->message && (strlen(errorDetails->message) > 0)) { - log_message(STORE_LOGLVL_DEBUG, " error details: %s", errorDetails->message); - } + //log_message(STORE_LOGLVL_DEBUG, "store_s3_complete_callback: request complete, status %d (%s)", status, S3_get_status_name(status)); + //if (errorDetails && errorDetails->message && (strlen(errorDetails->message) > 0)) { + // log_message(STORE_LOGLVL_DEBUG, " error details: %s", errorDetails->message); + //} rqst->result = status; rqst->error_details = errorDetails; } @@ -196,12 +196,8 @@ static int store_s3_tile_read(struct storage_backend *store, const char *xmlconf static struct stat_info store_s3_tile_stat(struct storage_backend *store, const char *xmlconfig, const char *options, int x, int y, int z) { struct store_s3_ctx *ctx = (struct store_s3_ctx*) store->storage_ctx; - char *path = NULL; - - log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_stat: Fetching tile properties"); - - path = malloc(PATH_MAX); + char *path = malloc(PATH_MAX); store_s3_xyz_to_storagekey(store, xmlconfig, options, x, y, z, path, PATH_MAX); log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_stat: getting properties for object %s", path); @@ -224,11 +220,16 @@ static struct stat_info store_s3_tile_stat(struct storage_backend *store, const struct stat_info tile_stat; if (request.result != S3StatusOK) { - const char *msg = ""; - if (request.error_details && request.error_details->message) { - msg = request.error_details->message; + if (request.result == S3StatusHttpErrorNotFound) { + // tile does not exist + log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_stat: tile not found in storage"); + } else { + const char *msg = ""; + if (request.error_details && request.error_details->message) { + msg = request.error_details->message; + } + log_message(STORE_LOGLVL_ERR, "store_s3_tile_stat: failed to retrieve object properties: %d (%s) %s", request.result, S3_get_status_name(request.result), msg); } - log_message(STORE_LOGLVL_ERR, "store_s3_tile_stat: failed to retrieve object properties: %d(%s)/%s", request.result, S3_get_status_name(request.result), msg); tile_stat.size = -1; tile_stat.expired = 0; tile_stat.mtime = 0; @@ -237,7 +238,7 @@ static struct stat_info store_s3_tile_stat(struct storage_backend *store, const return tile_stat; } - log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_stat: Read properties"); + log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_stat: successfully read properties"); tile_stat.size = request.tile_size; tile_stat.expired = request.tile_expired; From 1e91f168e406d8bfb889777d501d887db4a20b4b Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Wed, 3 Feb 2016 22:08:18 -0700 Subject: [PATCH 24/35] add s3 connection string description to example configuration file --- mod_tile.conf | 1 + src/store_s3.c | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/mod_tile.conf b/mod_tile.conf index 988e29a1..db5c07fd 100644 --- a/mod_tile.conf +++ b/mod_tile.conf @@ -14,6 +14,7 @@ LoadModule tile_module modules/mod_tile.so # The file based storage uses a simple file path as its storage path ( /path/to/tiledir ) # The RADOS based storage takes a location to the rados config file and a pool name ( rados://poolname/path/to/ceph.conf ) # The memcached based storage currently has no configuration options and always connects to memcached on localhost ( memcached:// ) +# The S3 based storage takes an access key, bucket, and path ( s3://access_key_id:secret_access_key[@host]/bucket_name[/basepath] ) # # The storage path can be overwritten on a style by style basis from the style TileConfigFile ModTileTileDir /var/lib/mod_tile diff --git a/src/store_s3.c b/src/store_s3.c index 74ac2c67..2d94db96 100644 --- a/src/store_s3.c +++ b/src/store_s3.c @@ -453,8 +453,6 @@ struct storage_backend* init_storage_s3(const char *connection_string) S3Status res; - log_message(STORE_LOGLVL_DEBUG, "init_storage_s3: initializing S3 storage backend for %s", connection_string); - if (!store || !ctx) { log_message(STORE_LOGLVL_ERR, "init_storage_s3: failed to allocate memory for context"); if (store) From e4b77ede220dfc53704d5e2ce731218e6ecb9497 Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Tue, 1 Mar 2016 16:42:29 -0700 Subject: [PATCH 25/35] fix double-free of storage object --- src/store_s3.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/store_s3.c b/src/store_s3.c index 2d94db96..b6704729 100644 --- a/src/store_s3.c +++ b/src/store_s3.c @@ -410,7 +410,7 @@ static int store_s3_close_storage(struct storage_backend *store) S3_deinitialize(); free(ctx); - free(store); + store->storage_ctx = NULL; store_s3_initialized = 0; return 0; From a625d0858071e8402dd8b9c212879a1a91e96821 Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Tue, 1 Mar 2016 15:49:31 -0700 Subject: [PATCH 26/35] made the s3 storage module a little less chatty --- src/store_s3.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/store_s3.c b/src/store_s3.c index b6704729..a3308df7 100644 --- a/src/store_s3.c +++ b/src/store_s3.c @@ -117,7 +117,7 @@ static int store_s3_tile_read(struct storage_backend *store, const char *xmlconf struct store_s3_ctx *ctx = (struct store_s3_ctx*) store->storage_ctx; char *path = malloc(PATH_MAX); - log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_read: fetching tile"); + //log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_read: fetching tile"); int tile_offset = store_s3_xyz_to_storagekey(store, xmlconfig, options, x, y, z, path, PATH_MAX); log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_read: retrieving object %s", path); @@ -199,7 +199,7 @@ static struct stat_info store_s3_tile_stat(struct storage_backend *store, const char *path = malloc(PATH_MAX); store_s3_xyz_to_storagekey(store, xmlconfig, options, x, y, z, path, PATH_MAX); - log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_stat: getting properties for object %s", path); + //log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_stat: getting properties for object %s", path); struct S3ResponseHandler responseHandler; responseHandler.propertiesCallback = &store_s3_properties_callback; @@ -216,13 +216,12 @@ static struct stat_info store_s3_tile_stat(struct storage_backend *store, const request.tile_size = 0; S3_head_object(ctx->ctx, path, NULL, &responseHandler, &request); - free(path); struct stat_info tile_stat; if (request.result != S3StatusOK) { if (request.result == S3StatusHttpErrorNotFound) { // tile does not exist - log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_stat: tile not found in storage"); + //log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_stat: tile not found in storage"); } else { const char *msg = ""; if (request.error_details && request.error_details->message) { @@ -235,16 +234,18 @@ static struct stat_info store_s3_tile_stat(struct storage_backend *store, const tile_stat.mtime = 0; tile_stat.atime = 0; tile_stat.ctime = 0; + free(path); return tile_stat; } - log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_stat: successfully read properties"); + log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_stat: successfully read properties of %s", path); tile_stat.size = request.tile_size; tile_stat.expired = request.tile_expired; tile_stat.mtime = request.tile_mod_time; tile_stat.atime = 0; tile_stat.ctime = 0; + free(path); return tile_stat; } From c13982498e91bdd1c1c6ec340d3944637424ce29 Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Mon, 7 Mar 2016 15:53:53 -0700 Subject: [PATCH 27/35] tweaks to logging --- src/store_s3.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/store_s3.c b/src/store_s3.c index a3308df7..f80f3fc6 100644 --- a/src/store_s3.c +++ b/src/store_s3.c @@ -66,7 +66,7 @@ static S3Status store_s3_properties_callback(const S3ResponseProperties *propert } } - log_message(STORE_LOGLVL_DEBUG, "store_s3_properties_callback: got properties for tile %s, length: %ld, content type: %s, expired: %d", rqst->path, rqst->tile_size, properties->contentType, rqst->tile_expired); + //log_message(STORE_LOGLVL_DEBUG, "store_s3_properties_callback: got properties for tile %s, length: %ld, content type: %s, expired: %d", rqst->path, rqst->tile_size, properties->contentType, rqst->tile_expired); return S3StatusOK; } @@ -227,7 +227,7 @@ static struct stat_info store_s3_tile_stat(struct storage_backend *store, const if (request.error_details && request.error_details->message) { msg = request.error_details->message; } - log_message(STORE_LOGLVL_ERR, "store_s3_tile_stat: failed to retrieve object properties: %d (%s) %s", request.result, S3_get_status_name(request.result), msg); + log_message(STORE_LOGLVL_ERR, "store_s3_tile_stat: failed to retrieve object properties for %s: %d (%s) %s", path, request.result, S3_get_status_name(request.result), msg); } tile_stat.size = -1; tile_stat.expired = 0; @@ -396,11 +396,11 @@ static int store_s3_metatile_expire(struct storage_backend *store, const char *x if (request.error_details && request.error_details->message) { msg = request.error_details->message; } - log_message(STORE_LOGLVL_ERR, "store_s3_metatile_expire: failed to update object: %d(%s)/%s", request.result, S3_get_status_name(request.result), msg); + log_message(STORE_LOGLVL_ERR, "store_s3_metatile_expire: failed to update object: %d (%s)/%s", request.result, S3_get_status_name(request.result), msg); return -1; } - log_message(STORE_LOGLVL_DEBUG, "store_s3_metatile_expire: Updated object metadata"); + log_message(STORE_LOGLVL_DEBUG, "store_s3_metatile_expire: updated object metadata"); return 0; } From 9e291c98dc400627b1da1c5ee9e6b79a192ef603 Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Thu, 26 May 2016 11:18:16 -0600 Subject: [PATCH 28/35] reduced logging --- src/store_s3.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/store_s3.c b/src/store_s3.c index f80f3fc6..a3d6ddbc 100644 --- a/src/store_s3.c +++ b/src/store_s3.c @@ -120,7 +120,7 @@ static int store_s3_tile_read(struct storage_backend *store, const char *xmlconf //log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_read: fetching tile"); int tile_offset = store_s3_xyz_to_storagekey(store, xmlconfig, options, x, y, z, path, PATH_MAX); - log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_read: retrieving object %s", path); + //log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_read: retrieving object %s", path); struct S3GetObjectHandler getObjectHandler; getObjectHandler.responseHandler.propertiesCallback = &store_s3_properties_callback; @@ -136,7 +136,6 @@ static int store_s3_tile_read(struct storage_backend *store, const char *xmlconf request.tile_size = 0; S3_get_object(ctx->ctx, path, NULL, 0, 0, NULL, &getObjectHandler, &request); - free(path); if (request.result != S3StatusOK) { const char *msg = ""; @@ -144,10 +143,15 @@ static int store_s3_tile_read(struct storage_backend *store, const char *xmlconf msg = request.error_details->message; } log_message(STORE_LOGLVL_ERR, "store_s3_tile_read: failed to retrieve object: %d(%s)/%s", request.result, S3_get_status_name(request.result), msg); + free(path); + path = NULL; return -1; } - log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_read: Read object of size %i", request.tile_size); + log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_read: retrieved metatile %s of size %i", path, request.tile_size); + + free(path); + path = NULL; // extract tile from metatile @@ -238,7 +242,7 @@ static struct stat_info store_s3_tile_stat(struct storage_backend *store, const return tile_stat; } - log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_stat: successfully read properties of %s", path); + //log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_stat: successfully read properties of %s", path); tile_stat.size = request.tile_size; tile_stat.expired = request.tile_expired; From 361187fca0a47b513dcfcdb0757b81ba98a71d1f Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Tue, 27 Sep 2016 09:12:02 -0600 Subject: [PATCH 29/35] fix errors in S3 usage --- src/store_s3.c | 99 ++++++++++++++++++++++++++------------------------ 1 file changed, 52 insertions(+), 47 deletions(-) diff --git a/src/store_s3.c b/src/store_s3.c index a3d6ddbc..7e87b59d 100644 --- a/src/store_s3.c +++ b/src/store_s3.c @@ -444,6 +444,22 @@ static char* url_decode(const char *src) } return dst; } + +static char* env_expand(const char *src) +{ + if (strstr(src, "${") == src && strrchr(src, '}') == (src + strlen(src) - 1)) { + char tmp[strlen(src) + 1]; + strcpy(tmp, src); + tmp[strlen(tmp) - 1] = '\0'; + char *val = getenv(tmp + 2); + if (NULL == val) { + log_message(STORE_LOGLVL_ERR, "init_storage_s3: environment variable %s not defined when initializing S3 configuration!", tmp + 2); + return NULL; + } + return val; + } + return src; +} #endif //Have libs3 struct storage_backend* init_storage_s3(const char *connection_string) @@ -494,6 +510,9 @@ struct storage_backend* init_storage_s3(const char *connection_string) if (strchr(fullurl, '@')) { bctx->secretAccessKey = strsep(&fullurl, "@"); bctx->hostName = strsep(&fullurl, "/"); + if (strlen(bctx->hostName) <= 0) { + bctx->hostName = NULL; + } } else { bctx->secretAccessKey = strsep(&fullurl, "/"); bctx->hostName = NULL; @@ -502,49 +521,37 @@ struct storage_backend* init_storage_s3(const char *connection_string) ctx->basepath = fullurl; - if (strstr(bctx->accessKeyId, "${") == bctx->accessKeyId && strrchr(bctx->accessKeyId, '}') == (bctx->accessKeyId + strlen(bctx->accessKeyId) - 1)) { - char tmp[strlen(bctx->accessKeyId) + 1]; - strcpy(tmp, bctx->accessKeyId); - tmp[strlen(tmp) - 1] = '\0'; - char *val = getenv(tmp + 2); - if (NULL == val) { - log_message(STORE_LOGLVL_ERR, "init_storage_s3: environment variable %s not defined when initializing S3 configuration!", tmp + 2); - free(ctx); - free(store); - return NULL; - } - bctx->accessKeyId = val; + bctx->accessKeyId = env_expand(bctx->accessKeyId); + if (bctx->accessKeyId == NULL) { + free(ctx); + free(store); + return NULL; } bctx->accessKeyId = url_decode(bctx->accessKeyId); - if (strstr(bctx->secretAccessKey, "${") == bctx->secretAccessKey && strrchr(bctx->secretAccessKey, '}') == (bctx->secretAccessKey + strlen(bctx->secretAccessKey) - 1)) { - char tmp[strlen(bctx->secretAccessKey) + 1]; - strcpy(tmp, bctx->secretAccessKey); - tmp[strlen(tmp) - 1] = '\0'; - char *val = getenv(tmp + 2); - if (NULL == val) { - log_message(STORE_LOGLVL_ERR, "init_storage_s3: environment variable %s not defined when initializing S3 configuration!", tmp + 2); - free(ctx); - free(store); - return NULL; - } - bctx->secretAccessKey = val; + bctx->secretAccessKey = env_expand(bctx->secretAccessKey); + if (bctx->secretAccessKey == NULL) { + free(ctx); + free(store); + return NULL; } bctx->secretAccessKey = url_decode(bctx->secretAccessKey); - bctx->hostName = url_decode(bctx->hostName); - if (strstr(bctx->bucketName, "${") == bctx->bucketName && strrchr(bctx->bucketName, '}') == (bctx->bucketName + strlen(bctx->bucketName) - 1)) { - char tmp[strlen(bctx->bucketName) + 1]; - strcpy(tmp, bctx->bucketName); - tmp[strlen(tmp) - 1] = '\0'; - char *val = getenv(tmp + 2); - if (NULL == val) { - log_message(STORE_LOGLVL_ERR, "init_storage_s3: environment variable %s not defined when initializing S3 configuration!", tmp + 2); + if (bctx->hostName) { + bctx->hostName = env_expand(bctx->hostName); + if (bctx->hostName == NULL) { free(ctx); free(store); return NULL; } - bctx->bucketName = val; + bctx->hostName = url_decode(bctx->hostName); + } + + bctx->bucketName = env_expand(bctx->bucketName); + if (bctx->bucketName == NULL) { + free(ctx); + free(store); + return NULL; } bctx->bucketName = url_decode(bctx->bucketName); @@ -552,23 +559,21 @@ struct storage_backend* init_storage_s3(const char *connection_string) bctx->securityToken = NULL; bctx->uriStyle = S3UriStyleVirtualHost; - if (strstr(ctx->basepath, "${") == ctx->basepath && strrchr(ctx->basepath, '}') == (ctx->basepath + strlen(ctx->basepath) - 1)) { - char tmp[strlen(ctx->basepath) + 1]; - strcpy(tmp, ctx->basepath); - tmp[strlen(tmp) - 1] = '\0'; - char *val = getenv(tmp + 2); - if (NULL == val) { - log_message(STORE_LOGLVL_ERR, "init_storage_s3: environment variable %s not defined when initializing S3 configuration!", tmp + 2); - free(ctx); - free(store); - return NULL; - } - ctx->basepath = val; + ctx->basepath = env_expand(ctx->basepath); + if (ctx->basepath == NULL) { + free(ctx); + free(store); + return NULL; } ctx->basepath = url_decode(ctx->basepath); - log_message(STORE_LOGLVL_DEBUG, "init_storage_s3 completed keyid: %s, key: %s, bucket: %s, basepath: %s", ctx->ctx->accessKeyId, ctx->ctx->secretAccessKey, ctx->ctx->bucketName, ctx->basepath); - + if (bctx->hostName) { + log_message(STORE_LOGLVL_DEBUG, "init_storage_s3 completed keyid: %s, key: %s, host: %s, bucket: %s, basepath: %s", + ctx->ctx->accessKeyId, ctx->ctx->secretAccessKey, ctx->ctx->hostName, ctx->ctx->bucketName, ctx->basepath); + } else { + log_message(STORE_LOGLVL_DEBUG, "init_storage_s3 completed keyid: %s, key: %s, bucket: %s, basepath: %s", + ctx->ctx->accessKeyId, ctx->ctx->secretAccessKey, ctx->ctx->bucketName, ctx->basepath); + } store->storage_ctx = ctx; store->tile_read = &store_s3_tile_read; From 27cb358cfe409191faef9390ad93a1988d310451 Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Fri, 30 Sep 2016 16:45:33 -0600 Subject: [PATCH 30/35] update to newer libs3 version that supports AWS Signature V4 --- src/store_s3.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/store_s3.c b/src/store_s3.c index 7e87b59d..0fe6841c 100644 --- a/src/store_s3.c +++ b/src/store_s3.c @@ -499,7 +499,8 @@ struct storage_backend* init_storage_s3(const char *connection_string) return NULL; } - // parse out the context information from the URL: s3://:[@]/[/] + // parse out the context information from the URL: + // s3://:[@]/[@region][/] struct S3BucketContext *bctx = ctx->ctx = malloc(sizeof(struct S3BucketContext)); char *fullurl = strdup(connection_string); @@ -507,7 +508,10 @@ struct storage_backend* init_storage_s3(const char *connection_string) // advance past "s3://" fullurl = &fullurl[5]; bctx->accessKeyId = strsep(&fullurl, ":"); - if (strchr(fullurl, '@')) { + char *nextSlash = strchr(fullurl, '/'); + char *nextAt = strchr(fullurl, '@'); + if ((nextAt != NULL) && (nextAt < nextSlash)) { + // there's an S3 host name in the URL bctx->secretAccessKey = strsep(&fullurl, "@"); bctx->hostName = strsep(&fullurl, "/"); if (strlen(bctx->hostName) <= 0) { @@ -517,7 +521,15 @@ struct storage_backend* init_storage_s3(const char *connection_string) bctx->secretAccessKey = strsep(&fullurl, "/"); bctx->hostName = NULL; } - bctx->bucketName = strsep(&fullurl, "/"); + + if (strchr(fullurl, '@')) { + // there's a region name with the bucket name + bctx->bucketName = strsep(&fullurl, "@"); + bctx->authRegion = strsep(&fullurl, "/"); + } + else { + bctx->bucketName = strsep(&fullurl, "/"); + } ctx->basepath = fullurl; From 3e0183d71cf7682ed04a63b9625d8cffe4de6e83 Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Fri, 30 Sep 2016 17:08:43 -0600 Subject: [PATCH 31/35] fix compile warning --- src/store_s3.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/store_s3.c b/src/store_s3.c index 0fe6841c..f435121e 100644 --- a/src/store_s3.c +++ b/src/store_s3.c @@ -445,7 +445,7 @@ static char* url_decode(const char *src) return dst; } -static char* env_expand(const char *src) +static const char* env_expand(const char *src) { if (strstr(src, "${") == src && strrchr(src, '}') == (src + strlen(src) - 1)) { char tmp[strlen(src) + 1]; From a7d7036d2a51d251a9994792fb016d176e9628be Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Sun, 2 Oct 2016 16:40:35 -0600 Subject: [PATCH 32/35] fix uninitialized memory --- src/store_s3.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/store_s3.c b/src/store_s3.c index f435121e..a858b29c 100644 --- a/src/store_s3.c +++ b/src/store_s3.c @@ -529,6 +529,7 @@ struct storage_backend* init_storage_s3(const char *connection_string) } else { bctx->bucketName = strsep(&fullurl, "/"); + bctx->authRegion = NULL; } ctx->basepath = fullurl; From 0a229e2b72a4f2180012d15a9cacc0ccd39a43f5 Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Mon, 3 Oct 2016 07:21:24 -0600 Subject: [PATCH 33/35] improve logging of different tile URL cases --- src/store_s3.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/store_s3.c b/src/store_s3.c index a858b29c..917cd9c7 100644 --- a/src/store_s3.c +++ b/src/store_s3.c @@ -580,12 +580,14 @@ struct storage_backend* init_storage_s3(const char *connection_string) } ctx->basepath = url_decode(ctx->basepath); - if (bctx->hostName) { - log_message(STORE_LOGLVL_DEBUG, "init_storage_s3 completed keyid: %s, key: %s, host: %s, bucket: %s, basepath: %s", - ctx->ctx->accessKeyId, ctx->ctx->secretAccessKey, ctx->ctx->hostName, ctx->ctx->bucketName, ctx->basepath); + if (bctx->hostName && bctx->authRegion) { + log_message(STORE_LOGLVL_DEBUG, "init_storage_s3 completed keyid: %s, key: %s, host: %s, region: %s, bucket: %s, basepath: %s", ctx->ctx->accessKeyId, ctx->ctx->secretAccessKey, ctx->ctx->hostName, ctx->ctx->authRegion, ctx->ctx->bucketName, ctx->basepath); + } else if (bctx->hostName) { + log_message(STORE_LOGLVL_DEBUG, "init_storage_s3 completed keyid: %s, key: %s, host: %s, bucket: %s, basepath: %s", ctx->ctx->accessKeyId, ctx->ctx->secretAccessKey, ctx->ctx->hostName, ctx->ctx->bucketName, ctx->basepath); + } else if (bctx->authRegion) { + log_message(STORE_LOGLVL_DEBUG, "init_storage_s3 completed keyid: %s, key: %s, region: %s, bucket: %s, basepath: %s", ctx->ctx->accessKeyId, ctx->ctx->secretAccessKey, ctx->ctx->authRegion, ctx->ctx->bucketName, ctx->basepath); } else { - log_message(STORE_LOGLVL_DEBUG, "init_storage_s3 completed keyid: %s, key: %s, bucket: %s, basepath: %s", - ctx->ctx->accessKeyId, ctx->ctx->secretAccessKey, ctx->ctx->bucketName, ctx->basepath); + log_message(STORE_LOGLVL_DEBUG, "init_storage_s3 completed keyid: %s, key: %s, bucket: %s, basepath: %s", ctx->ctx->accessKeyId, ctx->ctx->secretAccessKey, ctx->ctx->bucketName, ctx->basepath); } store->storage_ctx = ctx; From 33cc3cb168da516996297f6f7fee04e1b94a221c Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Mon, 3 Oct 2016 09:07:35 -0600 Subject: [PATCH 34/35] fix uninitialized property field and spelling error --- src/gen_tile_test.cpp | 4 ++-- src/store_s3.c | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/gen_tile_test.cpp b/src/gen_tile_test.cpp index 188d4a07..13c4740b 100644 --- a/src/gen_tile_test.cpp +++ b/src/gen_tile_test.cpp @@ -594,14 +594,14 @@ TEST_CASE( "renderd", "tile generation" ) { } SECTION("renderd startup unrecognized option", "should return 1") { - int ret = system("./renderd --doesnotexit"); + int ret = system("./renderd --doesnotexist"); ret = WEXITSTATUS(ret); //CAPTURE( ret ); REQUIRE( ret == 1 ); } SECTION("renderd startup invalid option", "should return 1") { - int ret = system("./renderd -doesnotexit"); + int ret = system("./renderd -doesnotexist"); ret = WEXITSTATUS(ret); //CAPTURE( ret ); REQUIRE( ret == 1 ); diff --git a/src/store_s3.c b/src/store_s3.c index 917cd9c7..46893d9e 100644 --- a/src/store_s3.c +++ b/src/store_s3.c @@ -389,6 +389,7 @@ static int store_s3_metatile_expire(struct storage_backend *store, const char *x props.md5 = NULL; props.metaDataCount = 1; props.metaData = &expireTag; + props.useServerSideEncryption = 0; int64_t lastModified; From cedc541be99c6a0ade29791eff4463ca697cc8dc Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Wed, 16 Nov 2016 08:27:17 -0700 Subject: [PATCH 35/35] improve error checking --- src/gen_tile_test.cpp | 2 +- src/store_s3.c | 85 +++++++++++++++++++++++++++++++++++-------- 2 files changed, 71 insertions(+), 16 deletions(-) diff --git a/src/gen_tile_test.cpp b/src/gen_tile_test.cpp index 188d4a07..9c037aa6 100644 --- a/src/gen_tile_test.cpp +++ b/src/gen_tile_test.cpp @@ -973,7 +973,7 @@ TEST_CASE("storage-backend/s3", "S3 tile storage backend") { const char *bucketpath = "mod_tile_test"; s3_connection_url = (char*) malloc(1024); - sprintf(s3_connection_url, "s3://%s:%s/%s/%s", keyid, accesskey, bucketname, bucketpath); + snprintf(s3_connection_url, 1024, "s3://%s:%s/%s/%s", keyid, accesskey, bucketname, bucketpath); SECTION("storage-backend/s3/initialise", "should return 1") { struct storage_backend *store = NULL; diff --git a/src/store_s3.c b/src/store_s3.c index 7e87b59d..93fb7636 100644 --- a/src/store_s3.c +++ b/src/store_s3.c @@ -38,6 +38,7 @@ struct s3_tile_request { struct store_s3_ctx { S3BucketContext* ctx; const char *basepath; + char *urlcopy; }; static int store_s3_xyz_to_storagekey(struct storage_backend *store, const char *xmlconfig, const char *options, int x, int y, int z, char *key, size_t keylen) @@ -76,8 +77,12 @@ S3Status store_s3_object_data_callback(int bufferSize, const char *buffer, void struct s3_tile_request *rqst = (struct s3_tile_request*) callbackData; if (rqst->cur_offset == 0 && rqst->tile == NULL) { - //log_message(STORE_LOGLVL_DEBUG, "store_s3_object_data_callback: allocating %ld byte buffer for tile", rqst->tile_size); + //log_message(STORE_LOGLVL_DEBUG, "store_s3_object_data_callback: allocating %z byte buffer for tile", rqst->tile_size); rqst->tile = malloc(rqst->tile_size); + if (NULL == rqst->tile) { + log_message(STORE_LOGLVL_ERR, "store_s3_object_data_callback: could not allocate %z byte buffer for tile!", rqst->tile_size); + return S3StatusOutOfMemory; + } } //log_message(STORE_LOGLVL_DEBUG, "store_s3_object_data_callback: appending %ld bytes to buffer, new offset %ld", bufferSize, rqst->cur_offset + bufferSize); @@ -201,7 +206,19 @@ static struct stat_info store_s3_tile_stat(struct storage_backend *store, const { struct store_s3_ctx *ctx = (struct store_s3_ctx*) store->storage_ctx; + struct stat_info tile_stat; + tile_stat.size = -1; + tile_stat.expired = 0; + tile_stat.mtime = 0; + tile_stat.atime = 0; + tile_stat.ctime = 0; + char *path = malloc(PATH_MAX); + if (NULL == path) { + log_message(STORE_LOGLVL_ERR, "store_s3_tile_stat: failed to allocate memory for tile path!"); + return tile_stat; + } + store_s3_xyz_to_storagekey(store, xmlconfig, options, x, y, z, path, PATH_MAX); //log_message(STORE_LOGLVL_DEBUG, "store_s3_tile_stat: getting properties for object %s", path); @@ -221,7 +238,6 @@ static struct stat_info store_s3_tile_stat(struct storage_backend *store, const S3_head_object(ctx->ctx, path, NULL, &responseHandler, &request); - struct stat_info tile_stat; if (request.result != S3StatusOK) { if (request.result == S3StatusHttpErrorNotFound) { // tile does not exist @@ -233,11 +249,6 @@ static struct stat_info store_s3_tile_stat(struct storage_backend *store, const } log_message(STORE_LOGLVL_ERR, "store_s3_tile_stat: failed to retrieve object properties for %s: %d (%s) %s", path, request.result, S3_get_status_name(request.result), msg); } - tile_stat.size = -1; - tile_stat.expired = 0; - tile_stat.mtime = 0; - tile_stat.atime = 0; - tile_stat.ctime = 0; free(path); return tile_stat; } @@ -247,8 +258,6 @@ static struct stat_info store_s3_tile_stat(struct storage_backend *store, const tile_stat.size = request.tile_size; tile_stat.expired = request.tile_expired; tile_stat.mtime = request.tile_mod_time; - tile_stat.atime = 0; - tile_stat.ctime = 0; free(path); return tile_stat; } @@ -414,6 +423,10 @@ static int store_s3_close_storage(struct storage_backend *store) struct store_s3_ctx *ctx = (struct store_s3_ctx*) store->storage_ctx; S3_deinitialize(); + if (NULL != ctx->urlcopy) { + free(ctx->urlcopy); + ctx->urlcopy = NULL; + } free(ctx); store->storage_ctx = NULL; store_s3_initialized = 0; @@ -445,7 +458,7 @@ static char* url_decode(const char *src) return dst; } -static char* env_expand(const char *src) +static const char* env_expand(const char *src) { if (strstr(src, "${") == src && strrchr(src, '}') == (src + strlen(src) - 1)) { char tmp[strlen(src) + 1]; @@ -469,10 +482,15 @@ struct storage_backend* init_storage_s3(const char *connection_string) "init_storage_s3: Support for libs3 and therefore S3 storage has not been compiled into this program"); return NULL; #else + if (strstr(connection_string, "s3://") != connection_string) { + log_message(STORE_LOGLVL_ERR, "init_storage_s3: connection string invalid for S3 storage!"); + return NULL; + } + struct storage_backend *store = malloc(sizeof(struct storage_backend)); struct store_s3_ctx *ctx = malloc(sizeof(struct store_s3_ctx)); - S3Status res; + S3Status res = S3StatusErrorUnknown; if (!store || !ctx) { log_message(STORE_LOGLVL_ERR, "init_storage_s3: failed to allocate memory for context"); @@ -485,12 +503,13 @@ struct storage_backend* init_storage_s3(const char *connection_string) pthread_mutex_lock(&qLock); if (!store_s3_initialized) { - log_message(STORE_LOGLVL_DEBUG, "init_storage_s3: global init of libs3", connection_string); + log_message(STORE_LOGLVL_DEBUG, "init_storage_s3: global init of libs3"); res = S3_initialize(NULL, S3_INIT_ALL, NULL); store_s3_initialized = 1; } else { res = S3StatusOK; } + pthread_mutex_unlock(&qLock); if (res != S3StatusOK) { log_message(STORE_LOGLVL_ERR, "init_storage_s3: failed to initialize S3 library: %s", S3_get_status_name(res)); @@ -502,22 +521,58 @@ struct storage_backend* init_storage_s3(const char *connection_string) // parse out the context information from the URL: s3://:[@]/[/] struct S3BucketContext *bctx = ctx->ctx = malloc(sizeof(struct S3BucketContext)); - char *fullurl = strdup(connection_string); + ctx->urlcopy = strdup(connection_string); + if (NULL == ctx->urlcopy) { + log_message(STORE_LOGLVL_ERR, "init_storage_s3: error allocating memory for connection string!"); + free(ctx); + free(store); + return NULL; + } // advance past "s3://" - fullurl = &fullurl[5]; + char *fullurl = &ctx->urlcopy[5]; bctx->accessKeyId = strsep(&fullurl, ":"); if (strchr(fullurl, '@')) { bctx->secretAccessKey = strsep(&fullurl, "@"); bctx->hostName = strsep(&fullurl, "/"); - if (strlen(bctx->hostName) <= 0) { + if (bctx->hostName != NULL && strlen(bctx->hostName) <= 0) { bctx->hostName = NULL; } } else { bctx->secretAccessKey = strsep(&fullurl, "/"); bctx->hostName = NULL; } + if (bctx->accessKeyId != NULL && strlen(bctx->accessKeyId) <= 0) { + bctx->accessKeyId = NULL; + } + if (bctx->secretAccessKey != NULL && strlen(bctx->secretAccessKey) <= 0) { + bctx->secretAccessKey = NULL; + } bctx->bucketName = strsep(&fullurl, "/"); + if (bctx->bucketName != NULL && strlen(bctx->bucketName) <= 0) { + bctx->bucketName = NULL; + } + + if (bctx->accessKeyId == NULL) { + log_message(STORE_LOGLVL_ERR, "init_storage_s3: S3 access key ID not provided in connection string!"); + free(ctx); + free(store); + return NULL; + } + + if (bctx->secretAccessKey == NULL) { + log_message(STORE_LOGLVL_ERR, "init_storage_s3: S3 secret access key not provided in connection string!"); + free(ctx); + free(store); + return NULL; + } + + if (bctx->bucketName == NULL) { + log_message(STORE_LOGLVL_ERR, "init_storage_s3: S3 bucket name not provided in connection string!"); + free(ctx); + free(store); + return NULL; + } ctx->basepath = fullurl;