diff --git a/CHANGELOG.md b/CHANGELOG.md index e31ec4cd..554c2e3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,10 @@ clcache changelog `CLCACHE_OBJECT_CACHE_TIMEOUT_MS` environment variable. * Improvement: Greatly improved concurrency of clcache such that concurrent invocations of the tool no longer block each other. + * Improvement: Improve hit rate when alternating between two identical + versions of the same source file that transitively get different contents of + the included files (a common case when switching back and forth between + branches). ## clcache 3.2.0 (2016-07-28) diff --git a/clcache.py b/clcache.py index e2dc30bd..fef54ef6 100644 --- a/clcache.py +++ b/clcache.py @@ -58,11 +58,11 @@ # to use it as mark for relative path. BASEDIR_REPLACEMENT = '?' +# ManifestEntry: an entry in a manifest file # `includeFiles`: list of paths to include files, which this source file uses -# `includesContentToObjectMap`: dictionary -# key: cumulative hash of all include files' content in includeFiles -# value: key in the cache, under which the object file is stored -Manifest = namedtuple('Manifest', ['includeFiles', 'includesContentToObjectMap']) +# `includesContentsHash`: hash of the contents of the includeFiles +# `objectHash`: hash of the object in cache +ManifestEntry = namedtuple('ManifestEntry', ['includeFiles', 'includesContentHash', 'objectHash']) CompilerArtifacts = namedtuple('CompilerArtifacts', ['objectFilePath', 'stdout', 'stderr']) @@ -108,10 +108,6 @@ class IncludeNotFoundException(Exception): pass -class IncludeChangedException(Exception): - pass - - class CacheLockException(Exception): pass @@ -125,6 +121,24 @@ def __str__(self): return repr(self.message) +class Manifest(object): + def __init__(self, entries=None): + if entries is None: + entries = [] + self._entries = entries.copy() + + def entries(self): + return self._entries + + def addEntry(self, entry): + """Adds entry at the top of the entries""" + self._entries.insert(0, entry) + + def touchEntry(self, entryIndex): + """Moves entry in entryIndex position to the top of entries()""" + self._entries.insert(0, self._entries.pop(entryIndex)) + + class ManifestSection(object): def __init__(self, manifestSectionDir): self.manifestSectionDir = manifestSectionDir @@ -137,10 +151,14 @@ def manifestFiles(self): return filesBeneath(self.manifestSectionDir) def setManifest(self, manifestHash, manifest): + manifestPath = self.manifestPath(manifestHash) + printTraceStatement("Writing manifest with manifestHash = {} to {}".format(manifestHash, manifestPath)) ensureDirectoryExists(self.manifestSectionDir) - with open(self.manifestPath(manifestHash), 'w') as outFile: + with open(manifestPath, 'w') as outFile: # Converting namedtuple to JSON via OrderedDict preserves key names and keys order - json.dump(manifest._asdict(), outFile, sort_keys=True, indent=2) + entries = [e._asdict() for e in manifest.entries()] + jsonobject = {'entries': entries} + json.dump(jsonobject, outFile, sort_keys=True, indent=2) def getManifest(self, manifestHash): fileName = self.manifestPath(manifestHash) @@ -149,7 +167,8 @@ def getManifest(self, manifestHash): try: with open(fileName, 'r') as inFile: doc = json.load(inFile) - return Manifest(doc['includeFiles'], doc['includesContentToObjectMap']) + return Manifest([ManifestEntry(e['includeFiles'], e['includesContentHash'], e['objectHash']) + for e in doc['entries']]) except IOError: return None @@ -172,7 +191,7 @@ class ManifestRepository(object): # invalidation, such that a manifest that was stored using the old format is not # interpreted using the new format. Instead the old file will not be touched # again due to a new manifest hash and is cleaned away after some time. - MANIFEST_FILE_FORMAT_VERSION = 4 + MANIFEST_FILE_FORMAT_VERSION = 5 def __init__(self, manifestsRootDir): self._manifestsRootDir = manifestsRootDir @@ -219,26 +238,19 @@ def getManifestHash(compilerBinary, commandLine, sourceFile): @staticmethod def getIncludesContentHashForFiles(includes): - listOfIncludesHashes = [] - includeMissing = False + listOfHashes = [] - for path in sorted(includes.keys()): + for path in includes: try: - fileHash = getFileHash(path) - if fileHash != includes[path]: - raise IncludeChangedException() - listOfIncludesHashes.append(fileHash) + listOfHashes.append(getFileHash(path)) except FileNotFoundError: - includeMissing = True - - if includeMissing: - raise IncludeNotFoundException() + raise IncludeNotFoundException - return ManifestRepository.getIncludesContentHashForHashes(listOfIncludesHashes) + return ManifestRepository.getIncludesContentHashForHashes(listOfHashes) @staticmethod - def getIncludesContentHashForHashes(listOfIncludesHashes): - return HashAlgorithm(','.join(listOfIncludesHashes).encode()).hexdigest() + def getIncludesContentHashForHashes(listOfHashes): + return HashAlgorithm(','.join(listOfHashes).encode()).hexdigest() class CacheLock(object): @@ -754,7 +766,8 @@ def getStringHash(dataString): return hasher.hexdigest() -def expandBasedirPlaceholder(path, baseDir): +def expandBasedirPlaceholder(path): + baseDir = normalizeBaseDir(os.environ.get('CLCACHE_BASEDIR')) if path.startswith(BASEDIR_REPLACEMENT): if not baseDir: raise LogicException('No CLCACHE_BASEDIR set, but found relative path ' + path) @@ -763,13 +776,17 @@ def expandBasedirPlaceholder(path, baseDir): return path -def collapseBasedirToPlaceholder(path, baseDir): - assert path == os.path.normcase(path) - assert baseDir == os.path.normcase(baseDir) - if path.startswith(baseDir): - return path.replace(baseDir, BASEDIR_REPLACEMENT, 1) - else: +def collapseBasedirToPlaceholder(path): + baseDir = normalizeBaseDir(os.environ.get('CLCACHE_BASEDIR')) + if baseDir is None: return path + else: + assert path == os.path.normcase(path) + assert baseDir == os.path.normcase(baseDir) + if path.startswith(baseDir): + return path.replace(baseDir, BASEDIR_REPLACEMENT, 1) + else: + return path def ensureDirectoryExists(path): @@ -1371,24 +1388,20 @@ def processCacheHit(cache, objectFile, cachekey): return 0, cachedArtifacts.stdout, cachedArtifacts.stderr, False -def createManifest(manifestHash, includePaths): - baseDir = normalizeBaseDir(os.environ.get('CLCACHE_BASEDIR')) - - includes = {path:getFileHash(path) for path in includePaths} - includesContentHash = ManifestRepository.getIncludesContentHashForFiles(includes) +def createManifestEntry(manifestHash, includePaths): + includesWithHash = {path:getFileHash(path) for path in includePaths} + includesContentHash = ManifestRepository.getIncludesContentHashForHashes(includesWithHash.values()) cachekey = CompilerArtifactsRepository.computeKeyDirect(manifestHash, includesContentHash) - # Create new manifest - if baseDir: - relocatableIncludePaths = { - collapseBasedirToPlaceholder(path, baseDir):contentHash - for path, contentHash in includes.items() - } - manifest = Manifest(relocatableIncludePaths, {}) - else: - manifest = Manifest(includes, {}) - manifest.includesContentToObjectMap[includesContentHash] = cachekey - return manifest, cachekey + safeIncludes = [collapseBasedirToPlaceholder(path) for path in includesWithHash.keys()] + return ManifestEntry(safeIncludes, includesContentHash, cachekey) + + +def createOrUpdateManifest(manifestSection, manifestHash, entry): + manifest = manifestSection.getManifest(manifestHash) or Manifest() + manifest.addEntry(entry) + manifestSection.setManifest(manifestHash, manifest) + return manifest def postprocessUnusableManifestMiss( @@ -1401,8 +1414,8 @@ def postprocessUnusableManifestMiss( returnCode, compilerOutput, compilerStderr = invokeRealCompiler(compiler, cmdLine, captureOutput=True) includePaths, compilerOutput = parseIncludesSet(compilerOutput, sourceFile, stripIncludes) - if returnCode == 0 and os.path.exists(objectFile): - manifest, cachekey = createManifest(manifestHash, includePaths) + entry = createManifestEntry(manifestHash, includePaths) + cachekey = entry.objectHash cleanupRequired = False section = cache.compilerArtifactsRepository.section(cachekey) @@ -1411,6 +1424,7 @@ def postprocessUnusableManifestMiss( if returnCode == 0 and os.path.exists(objectFile): artifacts = CompilerArtifacts(objectFile, compilerOutput, compilerStderr) cleanupRequired = addObjectToCache(stats, cache, section, cachekey, artifacts) + manifest = createOrUpdateManifest(manifestSection, manifestHash, entry) manifestSection.setManifest(manifestHash, manifest) return returnCode, compilerOutput, compilerStderr, cleanupRequired @@ -1551,7 +1565,6 @@ def processCompileRequest(cache, compiler, args): def processDirect(cache, objectFile, compiler, cmdLine, sourceFile): - baseDir = normalizeBaseDir(os.environ.get('CLCACHE_BASEDIR')) manifestHash = ManifestRepository.getManifestHash(compiler, cmdLine, sourceFile) manifestSection = cache.manifestRepository.section(manifestHash) with manifestSection.lock: @@ -1561,21 +1574,27 @@ def processDirect(cache, objectFile, compiler, cmdLine, sourceFile): cache, objectFile, manifestSection, manifestHash, sourceFile, compiler, cmdLine, Statistics.registerSourceChangedMiss) - # NOTE: command line options already included in hash for manifest name - try: - includesContentHash = ManifestRepository.getIncludesContentHashForFiles({ - expandBasedirPlaceholder(path, baseDir):contentHash - for path, contentHash in manifest.includeFiles.items() - }) - except IncludeChangedException: - return postprocessUnusableManifestMiss( - cache, objectFile, manifestSection, manifestHash, sourceFile, compiler, cmdLine, - Statistics.registerHeaderChangedMiss) - - cachekey = manifest.includesContentToObjectMap.get(includesContentHash) - assert cachekey is not None - - return getOrSetArtifacts(cache, cachekey, objectFile, compiler, cmdLine, Statistics.registerEvictedMiss) + for entryIndex, entry in enumerate(manifest.entries()): + # NOTE: command line options already included in hash for manifest name + try: + includesContentHash = ManifestRepository.getIncludesContentHashForFiles( + [expandBasedirPlaceholder(path) for path in entry.includeFiles]) + + if entry.includesContentHash == includesContentHash: + cachekey = entry.objectHash + assert cachekey is not None + # Move manifest entry to the top of the entries in the manifest + manifest.touchEntry(entryIndex) + manifestSection.setManifest(manifestHash, manifest) + + return getOrSetArtifacts( + cache, cachekey, objectFile, compiler, cmdLine, Statistics.registerEvictedMiss) + except IncludeNotFoundException: + pass + + return postprocessUnusableManifestMiss( + cache, objectFile, manifestSection, manifestHash, sourceFile, compiler, cmdLine, + Statistics.registerHeaderChangedMiss) def processNoDirect(cache, objectFile, compiler, cmdLine, environment): diff --git a/integrationtests.py b/integrationtests.py index cc485f68..a2b9c1f8 100644 --- a/integrationtests.py +++ b/integrationtests.py @@ -233,6 +233,328 @@ def testHitsSimple(self): newHits = stats.numCacheHits() self.assertEqual(newHits, oldHits + 1) + def testAlternatingHeadersHit(self): + with cd(os.path.join(ASSETS_DIR, "hits-and-misses")), tempfile.TemporaryDirectory() as tempDir: + cache = clcache.Cache(tempDir) + customEnv = dict(os.environ, CLCACHE_DIR=tempDir) + baseCmd = CLCACHE_CMD + ["/nologo", "/EHsc", "/c"] + + with cache.statistics as stats: + self.assertEqual(stats.numCacheHits(), 0) + self.assertEqual(stats.numCacheMisses(), 0) + self.assertEqual(stats.numCacheEntries(), 0) + + # VERSION 1 + with open('stable-source-with-alternating-header.h', 'w') as f: + f.write("#define VERSION 1\n") + subprocess.check_call(baseCmd + ["stable-source-with-alternating-header.cpp"], env=customEnv) + + with cache.statistics as stats: + self.assertEqual(stats.numCacheHits(), 0) + self.assertEqual(stats.numCacheMisses(), 1) + self.assertEqual(stats.numCacheEntries(), 1) + + # VERSION 2 + with open('stable-source-with-alternating-header.h', 'w') as f: + f.write("#define VERSION 2\n") + subprocess.check_call(baseCmd + ["stable-source-with-alternating-header.cpp"], env=customEnv) + + with cache.statistics as stats: + self.assertEqual(stats.numCacheHits(), 0) + self.assertEqual(stats.numCacheMisses(), 2) + self.assertEqual(stats.numCacheEntries(), 2) + + # VERSION 1 again + with open('stable-source-with-alternating-header.h', 'w') as f: + f.write("#define VERSION 1\n") + subprocess.check_call(baseCmd + ["stable-source-with-alternating-header.cpp"], env=customEnv) + + with cache.statistics as stats: + self.assertEqual(stats.numCacheHits(), 1) + self.assertEqual(stats.numCacheMisses(), 2) + self.assertEqual(stats.numCacheEntries(), 2) + + # VERSION 2 again + with open('stable-source-with-alternating-header.h', 'w') as f: + f.write("#define VERSION 1\n") + subprocess.check_call(baseCmd + ["stable-source-with-alternating-header.cpp"], env=customEnv) + + with cache.statistics as stats: + self.assertEqual(stats.numCacheHits(), 2) + self.assertEqual(stats.numCacheMisses(), 2) + self.assertEqual(stats.numCacheEntries(), 2) + + def testRemovedHeader(self): + with cd(os.path.join(ASSETS_DIR, "hits-and-misses")), tempfile.TemporaryDirectory() as tempDir: + cache = clcache.Cache(tempDir) + customEnv = dict(os.environ, CLCACHE_DIR=tempDir) + baseCmd = CLCACHE_CMD + ["/nologo", "/EHsc", "/c"] + + with cache.statistics as stats: + self.assertEqual(stats.numCacheHits(), 0) + self.assertEqual(stats.numCacheMisses(), 0) + self.assertEqual(stats.numCacheEntries(), 0) + + # VERSION 1 + with open('stable-source-with-alternating-header.h', 'w') as f: + f.write("#define VERSION 1\n") + subprocess.check_call(baseCmd + ["stable-source-with-alternating-header.cpp"], env=customEnv) + + with cache.statistics as stats: + self.assertEqual(stats.numCacheHits(), 0) + self.assertEqual(stats.numCacheMisses(), 1) + self.assertEqual(stats.numCacheEntries(), 1) + + # Remove header, trigger the compiler which should fail + os.remove('stable-source-with-alternating-header.h') + with self.assertRaises(subprocess.CalledProcessError): + subprocess.check_call(baseCmd + ["stable-source-with-alternating-header.cpp"], env=customEnv) + + with cache.statistics as stats: + self.assertEqual(stats.numCacheHits(), 0) + self.assertEqual(stats.numCacheMisses(), 2) + self.assertEqual(stats.numCacheEntries(), 1) + + # VERSION 1 again + with open('stable-source-with-alternating-header.h', 'w') as f: + f.write("#define VERSION 1\n") + subprocess.check_call(baseCmd + ["stable-source-with-alternating-header.cpp"], env=customEnv) + + with cache.statistics as stats: + self.assertEqual(stats.numCacheHits(), 1) + self.assertEqual(stats.numCacheMisses(), 2) + self.assertEqual(stats.numCacheEntries(), 1) + + # Remove header again, trigger the compiler which should fail + os.remove('stable-source-with-alternating-header.h') + with self.assertRaises(subprocess.CalledProcessError): + subprocess.check_call(baseCmd + ["stable-source-with-alternating-header.cpp"], env=customEnv) + + with cache.statistics as stats: + self.assertEqual(stats.numCacheHits(), 1) + self.assertEqual(stats.numCacheMisses(), 3) + self.assertEqual(stats.numCacheEntries(), 1) + + def testAlternatingTransitiveHeader(self): + with cd(os.path.join(ASSETS_DIR, "hits-and-misses")), tempfile.TemporaryDirectory() as tempDir: + cache = clcache.Cache(tempDir) + customEnv = dict(os.environ, CLCACHE_DIR=tempDir) + baseCmd = CLCACHE_CMD + ["/nologo", "/EHsc", "/c"] + + with cache.statistics as stats: + self.assertEqual(stats.numCacheHits(), 0) + self.assertEqual(stats.numCacheMisses(), 0) + self.assertEqual(stats.numCacheEntries(), 0) + + # VERSION 1 + with open('alternating-header.h', 'w') as f: + f.write("#define VERSION 1\n") + subprocess.check_call(baseCmd + ["stable-source-transitive-header.cpp"], env=customEnv) + + with cache.statistics as stats: + self.assertEqual(stats.numCacheHits(), 0) + self.assertEqual(stats.numCacheMisses(), 1) + self.assertEqual(stats.numCacheEntries(), 1) + + # VERSION 2 + with open('alternating-header.h', 'w') as f: + f.write("#define VERSION 2\n") + subprocess.check_call(baseCmd + ["stable-source-transitive-header.cpp"], env=customEnv) + + with cache.statistics as stats: + self.assertEqual(stats.numCacheHits(), 0) + self.assertEqual(stats.numCacheMisses(), 2) + self.assertEqual(stats.numCacheEntries(), 2) + + # VERSION 1 again + with open('alternating-header.h', 'w') as f: + f.write("#define VERSION 1\n") + subprocess.check_call(baseCmd + ["stable-source-transitive-header.cpp"], env=customEnv) + + with cache.statistics as stats: + self.assertEqual(stats.numCacheHits(), 1) + self.assertEqual(stats.numCacheMisses(), 2) + self.assertEqual(stats.numCacheEntries(), 2) + + # VERSION 2 again + with open('alternating-header.h', 'w') as f: + f.write("#define VERSION 1\n") + subprocess.check_call(baseCmd + ["stable-source-transitive-header.cpp"], env=customEnv) + + with cache.statistics as stats: + self.assertEqual(stats.numCacheHits(), 2) + self.assertEqual(stats.numCacheMisses(), 2) + self.assertEqual(stats.numCacheEntries(), 2) + + def testRemovedTransitiveHeader(self): + with cd(os.path.join(ASSETS_DIR, "hits-and-misses")), tempfile.TemporaryDirectory() as tempDir: + cache = clcache.Cache(tempDir) + customEnv = dict(os.environ, CLCACHE_DIR=tempDir) + baseCmd = CLCACHE_CMD + ["/nologo", "/EHsc", "/c"] + + with cache.statistics as stats: + self.assertEqual(stats.numCacheHits(), 0) + self.assertEqual(stats.numCacheMisses(), 0) + self.assertEqual(stats.numCacheEntries(), 0) + + # VERSION 1 + with open('alternating-header.h', 'w') as f: + f.write("#define VERSION 1\n") + subprocess.check_call(baseCmd + ["stable-source-transitive-header.cpp"], env=customEnv) + + with cache.statistics as stats: + self.assertEqual(stats.numCacheHits(), 0) + self.assertEqual(stats.numCacheMisses(), 1) + self.assertEqual(stats.numCacheEntries(), 1) + + # Remove header, trigger the compiler which should fail + os.remove('alternating-header.h') + with self.assertRaises(subprocess.CalledProcessError): + subprocess.check_call(baseCmd + ["stable-source-transitive-header.cpp"], env=customEnv) + + with cache.statistics as stats: + self.assertEqual(stats.numCacheHits(), 0) + self.assertEqual(stats.numCacheMisses(), 2) + self.assertEqual(stats.numCacheEntries(), 1) + + # VERSION 1 again + with open('alternating-header.h', 'w') as f: + f.write("#define VERSION 1\n") + subprocess.check_call(baseCmd + ["stable-source-transitive-header.cpp"], env=customEnv) + + with cache.statistics as stats: + self.assertEqual(stats.numCacheHits(), 1) + self.assertEqual(stats.numCacheMisses(), 2) + self.assertEqual(stats.numCacheEntries(), 1) + + # Remove header again, trigger the compiler which should fail + os.remove('alternating-header.h') + with self.assertRaises(subprocess.CalledProcessError): + subprocess.check_call(baseCmd + ["stable-source-transitive-header.cpp"], env=customEnv) + + with cache.statistics as stats: + self.assertEqual(stats.numCacheHits(), 1) + self.assertEqual(stats.numCacheMisses(), 3) + self.assertEqual(stats.numCacheEntries(), 1) + + def testAlternatingIncludeOrder(self): + with cd(os.path.join(ASSETS_DIR, "hits-and-misses")), tempfile.TemporaryDirectory() as tempDir: + cache = clcache.Cache(tempDir) + customEnv = dict(os.environ, CLCACHE_DIR=tempDir) + baseCmd = CLCACHE_CMD + ["/nologo", "/EHsc", "/c"] + + with open('A.h', 'w') as header: + header.write('#define A 1\n') + with open('B.h', 'w') as header: + header.write('#define B 1\n') + + with cache.statistics as stats: + self.assertEqual(stats.numCacheHits(), 0) + self.assertEqual(stats.numCacheMisses(), 0) + self.assertEqual(stats.numCacheEntries(), 0) + + # VERSION 1 + with open('stable-source-with-alternating-header.h', 'w') as f: + f.write('#include "A.h"\n') + f.write('#include "B.h"\n') + subprocess.check_call(baseCmd + ["stable-source-with-alternating-header.cpp"], env=customEnv) + + with cache.statistics as stats: + self.assertEqual(stats.numCacheHits(), 0) + self.assertEqual(stats.numCacheMisses(), 1) + self.assertEqual(stats.numCacheEntries(), 1) + + # VERSION 2 + with open('stable-source-with-alternating-header.h', 'w') as f: + f.write('#include "B.h"\n') + f.write('#include "A.h"\n') + subprocess.check_call(baseCmd + ["stable-source-with-alternating-header.cpp"], env=customEnv) + + with cache.statistics as stats: + self.assertEqual(stats.numCacheHits(), 0) + self.assertEqual(stats.numCacheMisses(), 2) + self.assertEqual(stats.numCacheEntries(), 2) + + # VERSION 1 again + with open('stable-source-with-alternating-header.h', 'w') as f: + f.write('#include "A.h"\n') + f.write('#include "B.h"\n') + subprocess.check_call(baseCmd + ["stable-source-with-alternating-header.cpp"], env=customEnv) + + with cache.statistics as stats: + self.assertEqual(stats.numCacheHits(), 1) + self.assertEqual(stats.numCacheMisses(), 2) + self.assertEqual(stats.numCacheEntries(), 2) + + # VERSION 2 again + with open('stable-source-with-alternating-header.h', 'w') as f: + f.write('#include "B.h"\n') + f.write('#include "A.h"\n') + subprocess.check_call(baseCmd + ["stable-source-with-alternating-header.cpp"], env=customEnv) + + with cache.statistics as stats: + self.assertEqual(stats.numCacheHits(), 2) + self.assertEqual(stats.numCacheMisses(), 2) + self.assertEqual(stats.numCacheEntries(), 2) + + def testRepeatedIncludes(self): + with cd(os.path.join(ASSETS_DIR, "hits-and-misses")), tempfile.TemporaryDirectory() as tempDir: + cache = clcache.Cache(tempDir) + customEnv = dict(os.environ, CLCACHE_DIR=tempDir) + baseCmd = CLCACHE_CMD + ["/nologo", "/EHsc", "/c"] + + with open('A.h', 'w') as header: + header.write('#define A 1\n') + with open('B.h', 'w') as header: + header.write('#define B 1\n') + + with cache.statistics as stats: + self.assertEqual(stats.numCacheHits(), 0) + self.assertEqual(stats.numCacheMisses(), 0) + self.assertEqual(stats.numCacheEntries(), 0) + + # VERSION 1 + with open('stable-source-with-alternating-header.h', 'w') as f: + f.write('#include "A.h"\n') + f.write('#include "A.h"\n') + subprocess.check_call(baseCmd + ["stable-source-with-alternating-header.cpp"], env=customEnv) + + with cache.statistics as stats: + self.assertEqual(stats.numCacheHits(), 0) + self.assertEqual(stats.numCacheMisses(), 1) + self.assertEqual(stats.numCacheEntries(), 1) + + # VERSION 2 + with open('stable-source-with-alternating-header.h', 'w') as f: + f.write('#include "A.h"\n') + subprocess.check_call(baseCmd + ["stable-source-with-alternating-header.cpp"], env=customEnv) + + with cache.statistics as stats: + self.assertEqual(stats.numCacheHits(), 0) + self.assertEqual(stats.numCacheMisses(), 2) + self.assertEqual(stats.numCacheEntries(), 2) + + # VERSION 1 again + with open('stable-source-with-alternating-header.h', 'w') as f: + f.write('#include "A.h"\n') + f.write('#include "A.h"\n') + subprocess.check_call(baseCmd + ["stable-source-with-alternating-header.cpp"], env=customEnv) + + with cache.statistics as stats: + self.assertEqual(stats.numCacheHits(), 1) + self.assertEqual(stats.numCacheMisses(), 2) + self.assertEqual(stats.numCacheEntries(), 2) + + # VERSION 2 again + with open('stable-source-with-alternating-header.h', 'w') as f: + f.write('#include "A.h"\n') + subprocess.check_call(baseCmd + ["stable-source-with-alternating-header.cpp"], env=customEnv) + + with cache.statistics as stats: + self.assertEqual(stats.numCacheHits(), 2) + self.assertEqual(stats.numCacheMisses(), 2) + self.assertEqual(stats.numCacheEntries(), 2) + class TestPrecompiledHeaders(unittest.TestCase): def testSampleproject(self): diff --git a/tests/integrationtests/hits-and-misses/.gitignore b/tests/integrationtests/hits-and-misses/.gitignore new file mode 100644 index 00000000..0c501438 --- /dev/null +++ b/tests/integrationtests/hits-and-misses/.gitignore @@ -0,0 +1,4 @@ +*.obj + +# written by the tests +stable-source-with-alternating-header.h diff --git a/tests/integrationtests/hits-and-misses/stable-source-transitive-header.cpp b/tests/integrationtests/hits-and-misses/stable-source-transitive-header.cpp new file mode 100644 index 00000000..e713db2a --- /dev/null +++ b/tests/integrationtests/hits-and-misses/stable-source-transitive-header.cpp @@ -0,0 +1,9 @@ +#include + +#include "transitive-header.h" + +int main() +{ + std::cout << "A C++ file we compile twice and expect a hit" << std::endl; + return 0; +} diff --git a/tests/integrationtests/hits-and-misses/stable-source-with-alternating-header.cpp b/tests/integrationtests/hits-and-misses/stable-source-with-alternating-header.cpp new file mode 100644 index 00000000..32031da4 --- /dev/null +++ b/tests/integrationtests/hits-and-misses/stable-source-with-alternating-header.cpp @@ -0,0 +1,9 @@ +#include + +#include "stable-source-with-alternating-header.h" + +int main() +{ + std::cout << "A C++ file we compile twice and expect a hit" << std::endl; + return 0; +} diff --git a/tests/integrationtests/hits-and-misses/transitive-header.h b/tests/integrationtests/hits-and-misses/transitive-header.h new file mode 100644 index 00000000..47aa41e8 --- /dev/null +++ b/tests/integrationtests/hits-and-misses/transitive-header.h @@ -0,0 +1 @@ +#include "alternating-header.h" diff --git a/unittests.py b/unittests.py index 3fecdc57..b704ea10 100644 --- a/unittests.py +++ b/unittests.py @@ -20,6 +20,7 @@ CompilerArtifactsRepository, Configuration, Manifest, + ManifestEntry, ManifestRepository, Statistics, ) @@ -208,6 +209,17 @@ def testHitCounts(self): class TestManifestRepository(unittest.TestCase): + entry1 = ManifestEntry([r'somepath\myinclude.h'], + "fdde59862785f9f0ad6e661b9b5746b7", + "a649723940dc975ebd17167d29a532f8") + entry2 = ManifestEntry([r'somepath\myinclude.h', r'moreincludes.h'], + "474e7fc26a592d84dfa7416c10f036c6", + "8771d7ebcf6c8bd57a3d6485f63e3a89") + # Size in (120, 240] bytes + manifest1 = Manifest([entry1]) + # Size in (120, 240] bytes + manifest2 = Manifest([entry2]) + def _getDirectorySize(self, dirPath): def filesize(path, filename): return os.stat(os.path.join(path, filename)).st_size @@ -269,28 +281,21 @@ def testStoreAndGetManifest(self): manifestsRootDir = os.path.join(ASSETS_DIR, "manifests") mm = ManifestRepository(manifestsRootDir) - manifest1 = Manifest([r'somepath\myinclude.h'], { - "fdde59862785f9f0ad6e661b9b5746b7": "a649723940dc975ebd17167d29a532f8" - }) - manifest2 = Manifest([r'somepath\myinclude.h', 'moreincludes.h'], { - "474e7fc26a592d84dfa7416c10f036c6": "8771d7ebcf6c8bd57a3d6485f63e3a89" - }) - ms1 = mm.section("8a33738d88be7edbacef48e262bbb5bc") ms2 = mm.section("0623305942d216c165970948424ae7d1") - ms1.setManifest("8a33738d88be7edbacef48e262bbb5bc", manifest1) - ms2.setManifest("0623305942d216c165970948424ae7d1", manifest2) + ms1.setManifest("8a33738d88be7edbacef48e262bbb5bc", TestManifestRepository.manifest1) + ms2.setManifest("0623305942d216c165970948424ae7d1", TestManifestRepository.manifest2) retrieved1 = ms1.getManifest("8a33738d88be7edbacef48e262bbb5bc") self.assertIsNotNone(retrieved1) - self.assertEqual(retrieved1.includesContentToObjectMap["fdde59862785f9f0ad6e661b9b5746b7"], - "a649723940dc975ebd17167d29a532f8") + retrieved1Entry = retrieved1.entries()[0] + self.assertEqual(retrieved1Entry, TestManifestRepository.entry1) retrieved2 = ms2.getManifest("0623305942d216c165970948424ae7d1") self.assertIsNotNone(retrieved2) - self.assertEqual(retrieved2.includesContentToObjectMap["474e7fc26a592d84dfa7416c10f036c6"], - "8771d7ebcf6c8bd57a3d6485f63e3a89") + retrieved2Entry = retrieved2.entries()[0] + self.assertEqual(retrieved2Entry, TestManifestRepository.entry2) def testNonExistingManifest(self): manifestsRootDir = os.path.join(ASSETS_DIR, "manifests") @@ -303,16 +308,10 @@ def testClean(self): manifestsRootDir = os.path.join(ASSETS_DIR, "manifests") mm = ManifestRepository(manifestsRootDir) - # Size in (120, 240] bytes - manifest1 = Manifest([r'somepath\myinclude.h'], { - "fdde59862785f9f0ad6e661b9b5746b7": "a649723940dc975ebd17167d29a532f8" - }) - # Size in (120, 240] bytes - manifest2 = Manifest([r'somepath\myinclude.h', 'moreincludes.h'], { - "474e7fc26a592d84dfa7416c10f036c6": "8771d7ebcf6c8bd57a3d6485f63e3a89" - }) - mm.section("8a33738d88be7edbacef48e262bbb5bc").setManifest("8a33738d88be7edbacef48e262bbb5bc", manifest1) - mm.section("0623305942d216c165970948424ae7d1").setManifest("0623305942d216c165970948424ae7d1", manifest2) + mm.section("8a33738d88be7edbacef48e262bbb5bc").setManifest("8a33738d88be7edbacef48e262bbb5bc", + TestManifestRepository.manifest1) + mm.section("0623305942d216c165970948424ae7d1").setManifest("0623305942d216c165970948424ae7d1", + TestManifestRepository.manifest2) cleaningResultSize = mm.clean(240) # Only one of those manifests can be left @@ -906,6 +905,40 @@ def testParseIncludesGerman(self): self.assertTrue(r'' not in includesSet) +class TestManifest(unittest.TestCase): + entry1 = ManifestEntry([r'somepath\myinclude.h'], + "fdde59862785f9f0ad6e661b9b5746b7", + "a649723940dc975ebd17167d29a532f8") + entry2 = ManifestEntry([r'somepath\myinclude.h', r'moreincludes.h'], + "474e7fc26a592d84dfa7416c10f036c6", + "8771d7ebcf6c8bd57a3d6485f63e3a89") + entries = [entry1, entry2] + + def testCreateEmpty(self): + manifest = Manifest() + self.assertFalse(manifest.entries()) + + def testCreateWithEntries(self): + manifest = Manifest(TestManifest.entries) + self.assertEqual(TestManifest.entries, manifest.entries()) + + + def testAddEntry(self): + manifest = Manifest(TestManifest.entries) + newEntry = ManifestEntry([r'somepath\myotherinclude.h'], + "474e7fc26a592d84dfa7416c10f036c6", + "8771d7ebcf6c8bd57a3d6485f63e3a89") + manifest.addEntry(newEntry) + self.assertEqual(newEntry, manifest.entries()[0]) + + + def testTouchEntry(self): + manifest = Manifest(TestManifest.entries) + self.assertEqual(TestManifest.entry1, manifest.entries()[0]) + manifest.touchEntry(1) + self.assertEqual(TestManifest.entry2, manifest.entries()[0]) + + if __name__ == '__main__': unittest.TestCase.longMessage = True unittest.main()