Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CCDB/include/CCDB/CcdbApi.h
Original file line number Diff line number Diff line change
Expand Up @@ -556,7 +556,7 @@ class CcdbApi //: public DatabaseInterface
* @param tcl The TClass object describing the serialized type
* @return raw pointer to created object
*/
void* downloadFilesystemContent(std::string const& fullUrl, std::type_info const& tinfo, std::map<string, string>* headers) const;
void* downloadFilesystemContent(std::string const& fullUrl, std::type_info const& tinfo, std::map<std::string, std::string>* headers) const;

// initialize the TGrid (Alien connection)
bool initTGrid() const;
Expand Down
68 changes: 34 additions & 34 deletions CCDB/src/CcdbApi.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,7 @@ void CcdbApi::updateMetaInformationInLocalFile(std::string const& filename, std:
*/
std::string sanitizeObjectName(const std::string& objectName)
{
string tmpObjectName = objectName;
std::string tmpObjectName = objectName;
tmpObjectName.erase(std::remove_if(tmpObjectName.begin(), tmpObjectName.end(),
[](auto const& c) -> bool { return (!std::isalnum(c) && c != '_' && c != '/' && c != '.'); }),
tmpObjectName.end());
Expand Down Expand Up @@ -431,7 +431,7 @@ int CcdbApi::storeAsBinaryFile(const char* buffer, size_t size, const std::strin
CURLcode res = CURL_LAST;

for (size_t hostIndex = 0; hostIndex < hostsPool.size() && res > 0; hostIndex++) {
string fullUrl = getFullUrlForStorage(curl, path, objectType, metadata, sanitizedStartValidityTimestamp, sanitizedEndValidityTimestamp, hostIndex);
std::string fullUrl = getFullUrlForStorage(curl, path, objectType, metadata, sanitizedStartValidityTimestamp, sanitizedEndValidityTimestamp, hostIndex);
LOG(debug3) << "Full URL Encoded: " << fullUrl;
/* what URL that receives this POST */
curl_easy_setopt(curl, CURLOPT_URL, fullUrl.c_str());
Expand Down Expand Up @@ -476,57 +476,57 @@ int CcdbApi::storeAsTFile(const TObject* rootObject, std::string const& path, st
return storeAsBinaryFile(img->data(), img->size(), info.getFileName(), info.getObjectType(), path, metadata, startValidityTimestamp, endValidityTimestamp, maxSize);
}

string CcdbApi::getFullUrlForStorage(CURL* curl, const string& path, const string& objtype,
const map<string, string>& metadata,
long startValidityTimestamp, long endValidityTimestamp, int hostIndex) const
std::string CcdbApi::getFullUrlForStorage(CURL* curl, const std::string& path, const std::string& objtype,
const std::map<std::string, std::string>& metadata,
long startValidityTimestamp, long endValidityTimestamp, int hostIndex) const
{
// Prepare timestamps
string startValidityString = getTimestampString(startValidityTimestamp < 0 ? getCurrentTimestamp() : startValidityTimestamp);
string endValidityString = getTimestampString(endValidityTimestamp < 0 ? getFutureTimestamp(60 * 60 * 24 * 1) : endValidityTimestamp);
std::string startValidityString = getTimestampString(startValidityTimestamp < 0 ? getCurrentTimestamp() : startValidityTimestamp);
std::string endValidityString = getTimestampString(endValidityTimestamp < 0 ? getFutureTimestamp(60 * 60 * 24 * 1) : endValidityTimestamp);
// Get url
string url = getHostUrl(hostIndex);
std::string url = getHostUrl(hostIndex);
// Build URL
string fullUrl = url + "/" + path + "/" + startValidityString + "/" + endValidityString + "/";
std::string fullUrl = url + "/" + path + "/" + startValidityString + "/" + endValidityString + "/";
// Add type as part of metadata
// we need to URL encode the object type, since in case it has special characters (like the "<", ">" for templated classes) it won't work otherwise
char* objtypeEncoded = curl_easy_escape(curl, objtype.c_str(), objtype.size());
fullUrl += "ObjectType=" + string(objtypeEncoded) + "/";
fullUrl += "ObjectType=" + std::string(objtypeEncoded) + "/";
curl_free(objtypeEncoded);
// Add general metadata
for (auto& kv : metadata) {
string mfirst = kv.first;
string msecond = kv.second;
std::string mfirst = kv.first;
std::string msecond = kv.second;
// same trick for the metadata as for the object type
char* mfirstEncoded = curl_easy_escape(curl, mfirst.c_str(), mfirst.size());
char* msecondEncoded = curl_easy_escape(curl, msecond.c_str(), msecond.size());
fullUrl += string(mfirstEncoded) + "=" + string(msecondEncoded) + "/";
fullUrl += std::string(mfirstEncoded) + "=" + std::string(msecondEncoded) + "/";
curl_free(mfirstEncoded);
curl_free(msecondEncoded);
}
return fullUrl;
}

// todo make a single method of the one above and below
string CcdbApi::getFullUrlForRetrieval(CURL* curl, const string& path, const map<string, string>& metadata, long timestamp, int hostIndex) const
std::string CcdbApi::getFullUrlForRetrieval(CURL* curl, const std::string& path, const std::map<std::string, std::string>& metadata, long timestamp, int hostIndex) const
{
if (mInSnapshotMode) {
return getSnapshotFile(mSnapshotTopPath, path);
}

// Prepare timestamps
string validityString = getTimestampString(timestamp < 0 ? getCurrentTimestamp() : timestamp);
std::string validityString = getTimestampString(timestamp < 0 ? getCurrentTimestamp() : timestamp);
// Get host url
string hostUrl = getHostUrl(hostIndex);
std::string hostUrl = getHostUrl(hostIndex);
// Build URL
string fullUrl = hostUrl + "/" + path + "/" + validityString + "/";
std::string fullUrl = hostUrl + "/" + path + "/" + validityString + "/";
// Add metadata
for (auto& kv : metadata) {
string mfirst = kv.first;
string msecond = kv.second;
std::string mfirst = kv.first;
std::string msecond = kv.second;
// trick for the metadata in case it contains special characters
char* mfirstEncoded = curl_easy_escape(curl, mfirst.c_str(), mfirst.size());
char* msecondEncoded = curl_easy_escape(curl, msecond.c_str(), msecond.size());
fullUrl += string(mfirstEncoded) + "=" + string(msecondEncoded) + "/";
fullUrl += std::string(mfirstEncoded) + "=" + std::string(msecondEncoded) + "/";
curl_free(mfirstEncoded);
curl_free(msecondEncoded);
}
Expand Down Expand Up @@ -755,7 +755,7 @@ bool CcdbApi::receiveObject(void* dataHolder, std::string const& path, std::map<
CURLcode curlResultCode = CURL_LAST;

for (size_t hostIndex = 0; hostIndex < hostsPool.size() && (responseCode >= 400 || curlResultCode > 0); hostIndex++) {
string fullUrl = getFullUrlForRetrieval(curlHandle, path, metadata, timestamp, hostIndex);
std::string fullUrl = getFullUrlForRetrieval(curlHandle, path, metadata, timestamp, hostIndex);
curl_easy_setopt(curlHandle, CURLOPT_URL, fullUrl.c_str());

curlResultCode = CURL_perform(curlHandle);
Expand Down Expand Up @@ -885,7 +885,7 @@ void CcdbApi::snapshot(std::string const& ccdbrootpath, std::string const& local
{
// query all subpaths to ccdbrootpath
const auto allfolders = getAllFolders(ccdbrootpath);
std::map<string, string> metadata;
std::map<std::string, std::string> metadata;
for (auto& folder : allfolders) {
retrieveBlob(folder, localDir, metadata, timestamp);
}
Expand Down Expand Up @@ -977,7 +977,7 @@ bool CcdbApi::initTGrid() const
return gGrid != nullptr;
}

void* CcdbApi::downloadFilesystemContent(std::string const& url, std::type_info const& tinfo, std::map<string, string>* headers) const
void* CcdbApi::downloadFilesystemContent(std::string const& url, std::type_info const& tinfo, std::map<std::string, std::string>* headers) const
{
if ((url.find("alien:/", 0) != std::string::npos) && !initTGrid()) {
return nullptr;
Expand Down Expand Up @@ -1016,7 +1016,7 @@ void* CcdbApi::interpretAsTMemFileAndExtract(char* contentptr, size_t contentsiz
}

// navigate sequence of URLs until TFile content is found; object is extracted and returned
void* CcdbApi::navigateURLsAndRetrieveContent(CURL* curl_handle, std::string const& url, std::type_info const& tinfo, std::map<string, string>* headers) const
void* CcdbApi::navigateURLsAndRetrieveContent(CURL* curl_handle, std::string const& url, std::type_info const& tinfo, std::map<std::string, std::string>* headers) const
{
// a global internal data structure that can be filled with HTTP header information
// static --> to avoid frequent alloc/dealloc as optimization
Expand Down Expand Up @@ -1164,7 +1164,7 @@ void* CcdbApi::retrieveFromTFile(std::type_info const& tinfo, std::string const&

CURL* curl_handle = curl_easy_init();
curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, mUniqueAgentID.c_str());
string fullUrl = getFullUrlForRetrieval(curl_handle, path, metadata, timestamp); // todo check if function still works correctly in case mInSnapshotMode
std::string fullUrl = getFullUrlForRetrieval(curl_handle, path, metadata, timestamp); // todo check if function still works correctly in case mInSnapshotMode
// if we are in snapshot mode we can simply open the file; extract the object and return
if (mInSnapshotMode) {
auto res = extractFromLocalFile(fullUrl, tinfo, headers);
Expand Down Expand Up @@ -1218,8 +1218,8 @@ std::string CcdbApi::list(std::string const& path, bool latestOnly, std::string
curl_easy_setopt(curl, CURLOPT_USERAGENT, mUniqueAgentID.c_str());

struct curl_slist* headers = nullptr;
headers = curl_slist_append(headers, (string("Accept: ") + returnFormat).c_str());
headers = curl_slist_append(headers, (string("Content-Type: ") + returnFormat).c_str());
headers = curl_slist_append(headers, (std::string("Accept: ") + returnFormat).c_str());
headers = curl_slist_append(headers, (std::string("Content-Type: ") + returnFormat).c_str());
if (createdNotAfter >= 0) {
headers = curl_slist_append(headers, ("If-Not-After: " + std::to_string(createdNotAfter)).c_str());
}
Expand All @@ -1230,7 +1230,7 @@ std::string CcdbApi::list(std::string const& path, bool latestOnly, std::string

curlSetSSLOptions(curl);

string fullUrl;
std::string fullUrl;
// Perform the request, res will get the return code
for (size_t hostIndex = 0; hostIndex < hostsPool.size() && res != CURLE_OK; hostIndex++) {
fullUrl = getHostUrl(hostIndex);
Expand Down Expand Up @@ -1290,7 +1290,7 @@ void CcdbApi::truncate(std::string const& path) const
CURLcode res;
stringstream fullUrl;
for (size_t i = 0; i < hostsPool.size(); i++) {
string url = getHostUrl(i);
std::string url = getHostUrl(i);
fullUrl << url << "/truncate/" << path;

curl = curl_easy_init();
Expand Down Expand Up @@ -1436,7 +1436,7 @@ std::map<std::string, std::string> CcdbApi::retrieveHeaders(std::string const& p
auto do_remote_header_call = [this, &path, &metadata, timestamp]() -> std::map<std::string, std::string> {
CURL* curl = curl_easy_init();
CURLcode res = CURL_LAST;
string fullUrl = getFullUrlForRetrieval(curl, path, metadata, timestamp);
std::string fullUrl = getFullUrlForRetrieval(curl, path, metadata, timestamp);
std::map<std::string, std::string> headers;

if (curl != nullptr) {
Expand Down Expand Up @@ -1632,12 +1632,12 @@ int CcdbApi::updateMetadata(std::string const& path, std::map<std::string, std::
fullUrl << "?";

for (auto& kv : metadata) {
string mfirst = kv.first;
string msecond = kv.second;
std::string mfirst = kv.first;
std::string msecond = kv.second;
// same trick for the metadata as for the object type
char* mfirstEncoded = curl_easy_escape(curl, mfirst.c_str(), mfirst.size());
char* msecondEncoded = curl_easy_escape(curl, msecond.c_str(), msecond.size());
fullUrl << string(mfirstEncoded) + "=" + string(msecondEncoded) + "&";
fullUrl << std::string(mfirstEncoded) + "=" + std::string(msecondEncoded) + "&";
curl_free(mfirstEncoded);
curl_free(msecondEncoded);
}
Expand Down Expand Up @@ -1728,7 +1728,7 @@ void CcdbApi::scheduleDownload(RequestContext& requestContext, size_t* requestCo

CURL* curl_handle = curl_easy_init();
curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, mUniqueAgentID.c_str());
string fullUrl = getFullUrlForRetrieval(curl_handle, requestContext.path, requestContext.metadata, requestContext.timestamp);
std::string fullUrl = getFullUrlForRetrieval(curl_handle, requestContext.path, requestContext.metadata, requestContext.timestamp);
curl_slist* options_list = nullptr;
initCurlHTTPHeaderOptionsForRetrieve(curl_handle, options_list, requestContext.timestamp, &requestContext.headers,
requestContext.etag, requestContext.createdNotAfter, requestContext.createdNotBefore);
Expand Down
4 changes: 2 additions & 2 deletions CCDB/test/testBasicCCDBManager.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@

using namespace o2::ccdb;

static string basePath;
static std::string basePath;
std::string ccdbUrl = "http://ccdb-test.cern.ch:8080";
bool hostReachable = false;

Expand All @@ -43,7 +43,7 @@ struct Fixture {
std::cout << "Is host reachable ? --> " << hostReachable << std::endl;
char hostname[_POSIX_HOST_NAME_MAX];
gethostname(hostname, _POSIX_HOST_NAME_MAX);
basePath = string("Test/") + hostname + "/pid" + getpid() + "/BasicCCDBManager/";
basePath = std::string("Test/") + hostname + "/pid" + getpid() + "/BasicCCDBManager/";
std::cout << "Path we will use in this test suite : " + basePath << std::endl;
}
~Fixture()
Expand Down
30 changes: 15 additions & 15 deletions CCDB/test/testCcdbApi.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,8 @@ using namespace o2::ccdb;
namespace utf = boost::unit_test;
namespace tt = boost::test_tools;

static string ccdbUrl;
static string basePath;
static std::string ccdbUrl;
static std::string basePath;
bool hostReachable = false;

/**
Expand All @@ -63,7 +63,7 @@ struct Fixture {
cout << "Is host reachable ? --> " << hostReachable << endl;
char hostname[_POSIX_HOST_NAME_MAX];
gethostname(hostname, _POSIX_HOST_NAME_MAX);
basePath = string("Test/TestCcdbApi/") + hostname + "/pid" + getpid() + "/";
basePath = std::string("Test/TestCcdbApi/") + hostname + "/pid" + getpid() + "/";
// Replace dashes by underscores to avoid problems in the creation of local directories
std::replace(basePath.begin(), basePath.end(), '-','_');
cout << "Path we will use in this test suite : " + basePath << endl;
Expand All @@ -72,7 +72,7 @@ struct Fixture {
{
if (hostReachable) {
CcdbApi api;
map<string, string> metadata;
std::map<std::string, std::string> metadata;
api.init(ccdbUrl);
api.truncate(basePath + "*");
cout << "Test data truncated (" << basePath << ")" << endl;
Expand Down Expand Up @@ -104,7 +104,7 @@ struct test_fixture {
~test_fixture() = default;

CcdbApi api;
map<string, string> metadata;
std::map<std::string, std::string> metadata;
};

BOOST_AUTO_TEST_CASE(storeTMemFile_test, *utf::precondition(if_reachable()))
Expand Down Expand Up @@ -153,7 +153,7 @@ BOOST_AUTO_TEST_CASE(store_retrieve_TMemFile_templated_test, *utf::precondition(
BOOST_CHECK(f.api.retrieveFromTFileAny<o2::utils::RootChain>(basePath + "CCDBPath", f.metadata) == nullptr);

// try to get the headers back and to find the metadata
map<string, string> md;
std::map<std::string, std::string> md;
path2 = f.api.retrieveFromTFileAny<o2::ccdb::IdPath>(basePath + "CCDBPath", f.metadata, -1, &md);
BOOST_CHECK_EQUAL(md.count("Hello"), 1);
BOOST_CHECK_EQUAL(md["Hello"], "World");
Expand Down Expand Up @@ -345,7 +345,7 @@ BOOST_AUTO_TEST_CASE(delete_test, *utf::precondition(if_reachable()))
BOOST_CHECK(h2 == nullptr);
}

void countItems(const string& s, int& countObjects, int& countSubfolders)
void countItems(const std::string& s, int& countObjects, int& countSubfolders)
{
countObjects = 0;
countSubfolders = 0;
Expand All @@ -368,7 +368,7 @@ BOOST_AUTO_TEST_CASE(list_test, *utf::precondition(if_reachable()))
test_fixture f;

// test non-empty top dir
string s = f.api.list("", "application/json"); // top dir
std::string s = f.api.list("", "application/json"); // top dir
long nbLines = std::count(s.begin(), s.end(), '\n') + 1;
BOOST_CHECK(nbLines > 5);

Expand Down Expand Up @@ -436,7 +436,7 @@ BOOST_AUTO_TEST_CASE(TestHeaderParsing)
BOOST_AUTO_TEST_CASE(TestFetchingHeaders, *utf::precondition(if_reachable()))
{
// first store the object
string objectPath = basePath + "objectETag";
std::string objectPath = basePath + "objectETag";
test_fixture f;
TH1F h1("objectETag", "objectETag", 100, 0, 99);
f.api.storeAsTFile(&h1, objectPath, f.metadata);
Expand All @@ -445,7 +445,7 @@ BOOST_AUTO_TEST_CASE(TestFetchingHeaders, *utf::precondition(if_reachable()))
std::string etag;
std::vector<std::string> headers;
std::vector<std::string> pfns;
string path = objectPath + "/" + std::to_string(getCurrentTimestamp());
std::string path = objectPath + "/" + std::to_string(getCurrentTimestamp());
auto updated = CcdbApi::getCCDBEntryHeaders("http://ccdb-test.cern.ch:8080/" + path, etag, headers);
BOOST_CHECK_EQUAL(updated, true);
BOOST_REQUIRE(headers.size() != 0);
Expand All @@ -462,7 +462,7 @@ BOOST_AUTO_TEST_CASE(TestRetrieveHeaders, *utf::precondition(if_reachable()))

TH1F h1("object1", "object1", 100, 0, 99);
cout << "storing object 1 in " << basePath << "Test" << endl;
map<string, string> metadata;
std::map<std::string, std::string> metadata;
metadata["custom"] = "whatever";
f.api.storeAsTFile(&h1, basePath + "Test", metadata);

Expand Down Expand Up @@ -498,7 +498,7 @@ BOOST_AUTO_TEST_CASE(TestUpdateMetadata, *utf::precondition(if_reachable()))
// upload an object
TH1F h1("object1", "object1", 100, 0, 99);
cout << "storing object 1 in " << basePath << "Test" << endl;
map<string, string> metadata;
std::map<std::string, std::string> metadata;
metadata["custom"] = "whatever";
metadata["id"] = "first";
f.api.storeAsTFile(&h1, basePath + "Test", metadata);
Expand All @@ -507,10 +507,10 @@ BOOST_AUTO_TEST_CASE(TestUpdateMetadata, *utf::precondition(if_reachable()))
std::map<std::string, std::string> headers = f.api.retrieveHeaders(basePath + "Test", metadata);
BOOST_CHECK(headers.count("custom") > 0);
BOOST_CHECK(headers.at("custom") == "whatever");
string firstID = headers.at("ETag");
std::string firstID = headers.at("ETag");
firstID.erase(std::remove(firstID.begin(), firstID.end(), '"'), firstID.end());

map<string, string> newMetadata;
std::map<std::string, std::string> newMetadata;
newMetadata["custom"] = "somethingelse";

// update the metadata and check
Expand All @@ -529,7 +529,7 @@ BOOST_AUTO_TEST_CASE(TestUpdateMetadata, *utf::precondition(if_reachable()))
// get id
cout << "get id" << endl;
headers = f.api.retrieveHeaders(basePath + "Test", metadata);
string secondID = headers.at("ETag");
std::string secondID = headers.at("ETag");
secondID.erase(std::remove(secondID.begin(), secondID.end(), '"'), secondID.end());

// update the metadata by id
Expand Down
10 changes: 5 additions & 5 deletions CCDB/test/testCcdbApiMultipleUrls.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@ using namespace o2::ccdb;
namespace utf = boost::unit_test;
namespace tt = boost::test_tools;

static string ccdbUrl;
static string basePath;
static std::string ccdbUrl;
static std::string basePath;
bool hostReachable = false;

/**
Expand All @@ -40,14 +40,14 @@ struct Fixture {
cout << "ccdb url: " << ccdbUrl << endl;
hostReachable = api.isHostReachable();
cout << "Is host reachable ? --> " << hostReachable << endl;
basePath = string("Test/pid") + getpid() + "/";
basePath = std::string("Test/pid") + getpid() + "/";
cout << "Path we will use in this test suite : " + basePath << endl;
}
~Fixture()
{
if (hostReachable) {
CcdbApi api;
map<string, string> metadata;
std::map<std::string, std::string> metadata;
api.init(ccdbUrl);
api.truncate(basePath + "*");
cout << "Test data truncated (" << basePath << ")" << endl;
Expand Down Expand Up @@ -79,7 +79,7 @@ struct test_fixture {
~test_fixture() = default;

CcdbApi api;
map<string, string> metadata;
std::map<std::string, std::string> metadata;
};

BOOST_AUTO_TEST_CASE(storeAndRetrieve, *utf::precondition(if_reachable()))
Expand Down
Loading
Loading