diff --git a/clcache.py b/clcache.py index e9d77f42..fe117214 100644 --- a/clcache.py +++ b/clcache.py @@ -97,8 +97,8 @@ def childDirectories(path, absolute=True): def normalizeBaseDir(baseDir): if baseDir: baseDir = os.path.normcase(baseDir) - if not baseDir.endswith(os.path.sep): - baseDir += os.path.sep + if baseDir.endswith(os.path.sep): + baseDir = baseDir[0:-1] return baseDir else: # Converts empty string to None @@ -227,11 +227,25 @@ def getManifestHash(compilerBinary, commandLine, sourceFile): compilerHash = getCompilerHash(compilerBinary) # NOTE: We intentionally do not normalize command line to include - # preprocessor options. In direct mode we do not perform - # preprocessing before cache lookup, so all parameters are important. - # One of the few exceptions to this rule is the /MP switch, which only - # defines how many compiler processes are running simultaneusly. + # preprocessor options. In direct mode we do not perform preprocessing + # before cache lookup, so all parameters are important. One of the few + # exceptions to this rule is the /MP switch, which only defines how many + # compiler processes are running simultaneusly. Arguments that specify + # the compiler where to find the source files are parsed to replace + # ocurrences of CLCACHE_BASEDIR by a placeholder. commandLine = [arg for arg in commandLine if not arg.startswith("/MP")] + arguments, inputFiles = CommandLineAnalyzer.parseArgumentsAndInputFiles(commandLine) + collapseBasedirInCmdPath = lambda path: collapseBasedirToPlaceholder(os.path.normcase(os.path.abspath(path))) + + commandLine = [] + argumentsWithPaths = ("AI", "I", "FU") + for k in sorted(arguments.keys()): + if k in argumentsWithPaths: + commandLine.extend(["/" + k + collapseBasedirInCmdPath(arg) for arg in arguments[k]]) + else: + commandLine.extend(["/" + k + arg for arg in arguments[k]]) + + commandLine.extend(collapseBasedirInCmdPath(arg) for arg in inputFiles) additionalData = "{}|{}|{}".format( compilerHash, commandLine, ManifestRepository.MANIFEST_FILE_FORMAT_VERSION) diff --git a/integrationtests.py b/integrationtests.py index 0cbe38ed..4dc9c809 100644 --- a/integrationtests.py +++ b/integrationtests.py @@ -24,7 +24,7 @@ PYTHON_BINARY = sys.executable CLCACHE_SCRIPT = os.path.join(os.path.dirname(os.path.realpath(__file__)), "clcache.py") -ASSETS_DIR = os.path.join("tests", "integrationtests") +ASSETS_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "tests", "integrationtests") # pytest-cov note: subprocesses are coverage tested by default with some limitations # "For subprocess measurement environment variables must make it from the main process to the @@ -1073,37 +1073,102 @@ def testHitsViaMpConcurrent(self): class TestBasedir(unittest.TestCase): - def testBasedir(self): - with cd(os.path.join(ASSETS_DIR, "basedir")), tempfile.TemporaryDirectory() as tempDir: - # First, create two separate build directories with the same sources - for buildDir in ["builddir_a", "builddir_b"]: - shutil.rmtree(buildDir, ignore_errors=True) - os.mkdir(buildDir) + def setUp(self): + self.projectDir = os.path.join(ASSETS_DIR, "basedir") + self.tempDir = tempfile.TemporaryDirectory() + self.clcacheDir = os.path.join(self.tempDir.name, "clcache") + self.savedCwd = os.getcwd() + + os.chdir(self.tempDir.name) + + # First, create two separate build directories with the same sources + for buildDir in ["builddir_a", "builddir_b"]: + shutil.copytree(self.projectDir, buildDir) + + self.cache = clcache.Cache(self.clcacheDir) + + def tearDown(self): + os.chdir(self.savedCwd) + self.tempDir.cleanup() + + def _runCompiler(self, cppFile, extraArgs=None): + cmd = CLCACHE_CMD + ["/nologo", "/EHsc", "/c"] + if extraArgs: + cmd.extend(extraArgs) + cmd = cmd + [cppFile] + env = dict(os.environ, CLCACHE_DIR=self.clcacheDir, CLCACHE_BASEDIR=os.getcwd()) + self.assertEqual(subprocess.call(cmd, env=env), 0) + + def expectHit(self, runCompiler): + # Build once in one directory + with cd("builddir_a"): + runCompiler[0]() + with self.cache.statistics as stats: + self.assertEqual(stats.numCacheMisses(), 1) + self.assertEqual(stats.numCacheHits(), 0) - shutil.copy("main.cpp", buildDir) - shutil.copy("constants.h", buildDir) + # Build again in a different directory, this should hit now because of CLCACHE_BASEDIR + with cd("builddir_b"): + runCompiler[1]() + with self.cache.statistics as stats: + self.assertEqual(stats.numCacheMisses(), 1) + self.assertEqual(stats.numCacheHits(), 1) - cache = clcache.Cache(tempDir) + def expectMiss(self, runCompiler): + # Build once in one directory + with cd("builddir_a"): + runCompiler[0]() + with self.cache.statistics as stats: + self.assertEqual(stats.numCacheMisses(), 1) + self.assertEqual(stats.numCacheHits(), 0) + + # Build again in a different directory, this should hit now because of CLCACHE_BASEDIR + with cd("builddir_b"): + runCompiler[1]() + with self.cache.statistics as stats: + self.assertEqual(stats.numCacheMisses(), 2) + self.assertEqual(stats.numCacheHits(), 0) + + def testBasedirRelativePaths(self): + def runCompiler(): + self._runCompiler("main.cpp") + self.expectHit([runCompiler, runCompiler]) + + def testBasedirAbsolutePaths(self): + def runCompiler(): + self._runCompiler(os.path.join(os.getcwd(), "main.cpp")) + self.expectHit([runCompiler, runCompiler]) + + def testBasedirIncludeArg(self): + def runCompiler(): + self._runCompiler("main.cpp", ["/I{}".format(os.getcwd())]) + self.expectHit([runCompiler, runCompiler]) + + def testBasedirIncludeSlashes(self): + runCompiler1 = lambda: self._runCompiler("main.cpp", ["/I{}/".format(os.getcwd())]) + runCompiler2 = lambda: self._runCompiler("main.cpp", ["/I{}".format(os.getcwd())]) + self.expectHit([runCompiler1, runCompiler2]) + + def testBasedirIncludeArgDifferentCapitalization(self): + def runCompiler(): + self._runCompiler("main.cpp", ["/I{}".format(os.getcwd().upper())]) + self.expectHit([runCompiler, runCompiler]) + + def testBasedirDefineArg(self): + def runCompiler(): + self._runCompiler("main.cpp", ["/DRESOURCES_DIR={}".format(os.getcwd())]) + self.expectMiss([runCompiler, runCompiler]) + + def testBasedirRelativeIncludeArg(self): + basedir = os.getcwd() + + def runCompiler(cppFile="main.cpp"): + cmd = CLCACHE_CMD + ["/nologo", "/EHsc", "/c", "/I."] + cmd = cmd + [cppFile] + env = dict(os.environ, CLCACHE_DIR=self.clcacheDir, CLCACHE_BASEDIR=basedir) + self.assertEqual(subprocess.call(cmd, env=env), 0) - cmd = CLCACHE_CMD + ["/nologo", "/EHsc", "/c", "main.cpp"] - - # Build once in one directory - with cd("builddir_a"): - env = dict(os.environ, CLCACHE_DIR=tempDir, CLCACHE_BASEDIR=os.getcwd()) - self.assertEqual(subprocess.call(cmd, env=env), 0) - with cache.statistics as stats: - self.assertEqual(stats.numCacheMisses(), 1) - self.assertEqual(stats.numCacheHits(), 0) - - shutil.rmtree("builddir_a", ignore_errors=True) - - # Build again in a different directory, this should hit now because of CLCACHE_BASEDIR - with cd("builddir_b"): - env = dict(os.environ, CLCACHE_DIR=tempDir, CLCACHE_BASEDIR=os.getcwd()) - self.assertEqual(subprocess.call(cmd, env=env), 0) - with cache.statistics as stats: - self.assertEqual(stats.numCacheMisses(), 1) - self.assertEqual(stats.numCacheHits(), 1) + self.expectMiss([runCompiler, runCompiler]) class TestCleanCache(unittest.TestCase): diff --git a/unittests.py b/unittests.py index 03058c3d..97d1e5e2 100644 --- a/unittests.py +++ b/unittests.py @@ -69,14 +69,14 @@ def testNormalizeBaseDir(self): # Note: raw string literals cannot end in an odd number of backslashes # https://docs.python.org/3/faq/design.html#why-can-t-raw-strings-r-strings-end-with-a-backslash # So we consistenly use basic literals - self.assertEqual(clcache.normalizeBaseDir("c:"), "c:\\") - self.assertEqual(clcache.normalizeBaseDir("c:\\projects"), "c:\\projects\\") + self.assertEqual(clcache.normalizeBaseDir("c:"), "c:") + self.assertEqual(clcache.normalizeBaseDir("c:\\projects"), "c:\\projects") - self.assertEqual(clcache.normalizeBaseDir("C:\\"), "c:\\") - self.assertEqual(clcache.normalizeBaseDir("C:\\Projects\\"), "c:\\projects\\") + self.assertEqual(clcache.normalizeBaseDir("C:\\"), "c:") + self.assertEqual(clcache.normalizeBaseDir("C:\\Projects\\"), "c:\\projects") - self.assertEqual(clcache.normalizeBaseDir("c:\\projects with space"), "c:\\projects with space\\") - self.assertEqual(clcache.normalizeBaseDir("c:\\projects with ö"), "c:\\projects with ö\\") + self.assertEqual(clcache.normalizeBaseDir("c:\\projects with space"), "c:\\projects with space") + self.assertEqual(clcache.normalizeBaseDir("c:\\projects with ö"), "c:\\projects with ö") def testFilesBeneathSimple(self): with cd(os.path.join(ASSETS_DIR, "files-beneath")):