Merge pull request #443 from jkloetzke/py3.10

Python 3.10 compatibility

Author: jkloetzke

.github/workflows/workflow.yaml

@@ -5,9 +5,10 @@ on: [push, pull_request]
 jobs:
     build:
         runs-on: ubuntu-latest
+        timeout-minutes: 15
         strategy:
             matrix:
-                python-version: [3.6, 3.7, 3.8, 3.9]
+                python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
             fail-fast: false
 
         steps:

pym/bob/archive.py

@@ -23,7 +23,7 @@
 from .errors import BuildError
 from .tty import stepAction, stepMessage, \
     SKIPPED, EXECUTED, WARNING, INFO, TRACE, ERROR, IMPORTANT
-from .utils import asHexStr, removePath, isWindows
+from .utils import asHexStr, removePath, isWindows, sslNoVerifyContext
 from shlex import quote
 from tempfile import mkstemp, NamedTemporaryFile, TemporaryFile
 import argparse
@@ -676,7 +676,7 @@ def _getConnection(self):
         if url.scheme == 'http':
             connection = http.client.HTTPConnection(url.hostname, url.port)
         elif url.scheme == 'https':
-            ctx = None if self.__sslVerify else ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+            ctx = None if self.__sslVerify else sslNoVerifyContext()
             connection = http.client.HTTPSConnection(url.hostname, url.port,
                                                      context=ctx)
         else:

pym/bob/cmds/build/build.py

@@ -261,12 +261,12 @@ def _downloadLayerArgument(arg):
             setTui(args.jobs)
             builder.enableBufferedIO()
         try:
-            builder.cook(backlog, True if args.build_mode == 'checkout-only' else False)
+            builder.cook(backlog, True if args.build_mode == 'checkout-only' else False, loop)
             for p in backlog:
                 resultPath = p.getWorkspacePath()
                 if resultPath not in results:
                     results.append(resultPath)
-            builder.cook(providedBacklog, True if args.build_mode == 'checkout-only' else False, 1)
+            builder.cook(providedBacklog, True if args.build_mode == 'checkout-only' else False, loop, 1)
             for p in providedBacklog:
                 resultPath = p.getWorkspacePath()
                 if resultPath not in results:

pym/bob/cmds/build/builder.py

@@ -665,9 +665,9 @@ async def _runLocalSCMs(self, step, logger):
     def getStatistic(self):
         return self.__statistic
 
-    def __createGenericTask(self, coro):
+    def __createGenericTask(self, loop, coro):
         """Create and return task for coroutine."""
-        return asyncio.get_event_loop().create_task(self.__taskWrapper(coro))
+        return loop.create_task(self.__taskWrapper(coro))
 
     def __createCookTask(self, coro, step, checkoutOnly, tracker, count):
         """Create and return task for a cook()-like coroutine.
@@ -756,7 +756,7 @@ async def __taskWrapper(self, coro, trackingKey=None, tracker=None,
             self.__buildErrors.append(e)
             raise CancelBuildException
 
-    def cook(self, steps, checkoutOnly, depth=0):
+    def cook(self, steps, checkoutOnly, loop, depth=0):
         def cancelJobs():
             if self.__jobs > 1:
                 log("Cancel all running jobs...", WARNING)
@@ -767,19 +767,18 @@ def cancelJobs():
         async def dispatcher():
             if self.__jobs > 1:
                 packageJobs = [
-                    self.__createGenericTask(lambda s=step: self._cookTask(s, checkoutOnly, depth))
+                    self.__createGenericTask(loop, lambda s=step: self._cookTask(s, checkoutOnly, depth))
                     for step in steps ]
                 await gatherTasks(packageJobs)
             else:
                 packageJobs = []
                 for step in steps:
-                    job = self.__createGenericTask(lambda s=step: self._cookTask(s, checkoutOnly, depth))
+                    job = self.__createGenericTask(loop, lambda s=step: self._cookTask(s, checkoutOnly, depth))
                     packageJobs.append(job)
                     await asyncio.wait({job})
                 # retrieve results as last step to --keep-going
                 for job in packageJobs: job.result()
 
-        loop = asyncio.get_event_loop()
         self.__restart = True
         while self.__restart:
             self.__running = True
@@ -801,7 +800,7 @@ async def dispatcher():
             self.__tasksDone = 0
             self.__tasksNum = 0
 
-            j = self.__createGenericTask(dispatcher)
+            j = self.__createGenericTask(loop, dispatcher)
             try:
                 loop.add_signal_handler(signal.SIGINT, cancelJobs)
             except NotImplementedError:

pym/bob/cmds/graph.py

@@ -6,7 +6,7 @@
 from .. import BOB_VERSION
 from ..input import RecipeSet
 from ..tty import colorize
-from ..utils import processDefines
+from ..utils import processDefines, runInEventLoop
 import argparse
 import asyncio
 import json
@@ -15,9 +15,6 @@
 import collections
 from shutil import copyfile
 
-def run(coro):
-    return asyncio.get_event_loop().run_until_complete(coro)
-
 def findPackages(package, excludes, highlights, done, maxdepth, donePackages, level = 0):
     def isHighLigh(package):
         for h in highlights:
@@ -472,7 +469,7 @@ def getHover(package, options):
 simulation.force("link").links(links)
 </script>
 <div class='info' style="width: 100%; text-align: center;">
-  <div id='innerLeft' style="float: left"> Recipes: """ + run(RecipeSet().getScmStatus()) + """</div>
+  <div id='innerLeft' style="float: left"> Recipes: """ + runInEventLoop(RecipeSet().getScmStatus()) + """</div>
   <div id='innerRight' style="float: right">Bob version: """ + BOB_VERSION +"""</div>
   <div id='innerMiddle' style="display: inline-block">Generated using <a href="https://www.d3js.org">D3JS</a></div>
 </div>

pym/bob/cmds/jenkins.py

@@ -11,7 +11,7 @@
 from ..state import BobState
 from ..stringparser import isTrue
 from ..tty import WarnOnce
-from ..utils import asHexStr, processDefines
+from ..utils import asHexStr, processDefines, runInEventLoop, sslNoVerifyContext
 from shlex import quote
 import argparse
 import ast
@@ -67,9 +67,6 @@
 
 SHARED_GENERATION = '-2'
 
-def run(coro):
-    return asyncio.get_event_loop().run_until_complete(coro)
-
 class Wiper:
     def __init__(self):
         self.mode = 0
@@ -236,7 +233,7 @@ def getDescription(self, date, warnLazy):
         description.extend([
             "<h2>Recipe</h2>",
             "<p>Name: " + self.__recipe.getName()
-                + "<br/>Source: " + run(self.__recipe.getRecipeSet().getScmStatus())
+                + "<br/>Source: " + runInEventLoop(self.__recipe.getRecipeSet().getScmStatus())
                 + "<br/>Configured: " + date
                 + "<br/>Bob version: " + BOB_VERSION + "</p>",
             "<h2>Packages</h2>", "<ul>"
@@ -386,7 +383,7 @@ def dumpStepAuditGen(self, step, auditMeta):
         ]
         for (var, val) in sorted(step.getPackage().getMetaEnv().items()):
             cmd.extend(["-E", var, quote(val)])
-        recipesAudit = run(step.getPackage().getRecipe().getRecipeSet().getScmAudit())
+        recipesAudit = runInEventLoop(step.getPackage().getRecipe().getRecipeSet().getScmAudit())
         if recipesAudit is not None:
             cmd.extend(["--recipes",
                 quote(json.dumps(recipesAudit.dump(), sort_keys=True))])
@@ -1568,7 +1565,7 @@ def __init__(self, config, sslVerify):
         # Optionally disable SSL certificate checks
         if not sslVerify:
             handlers.append(urllib.request.HTTPSHandler(
-                context=ssl.SSLContext(ssl.PROTOCOL_SSLv23)))
+                context=sslNoVerifyContext()))
 
         # handle authorization
         username = url.get("username")

pym/bob/input.py

@@ -3302,8 +3302,11 @@ def __loadPlugin(self, mangledName, fileName, name):
         pluginStat = binStat(fileName)
         try:
             from importlib.machinery import SourceFileLoader
+            from importlib.util import spec_from_loader, module_from_spec
             loader = SourceFileLoader(mangledName, fileName)
-            mod = loader.load_module()
+            spec = spec_from_loader(mangledName, loader)
+            mod = module_from_spec(spec)
+            loader.exec_module(mod)
         except SyntaxError as e:
             import traceback
             raise ParseError("Error loading plugin "+fileName+": "+str(e),

pym/bob/scm/url.py

@@ -6,7 +6,7 @@
 from .. import BOB_VERSION
 from ..errors import BuildError, ParseError
 from ..stringparser import IfExpression
-from ..utils import asHexStr, hashFile, isWindows, removeUserFromUrl
+from ..utils import asHexStr, hashFile, isWindows, removeUserFromUrl, sslNoVerifyContext
 from .scm import Scm, ScmAudit
 import asyncio
 import concurrent
@@ -220,7 +220,7 @@ def getProperties(self, isJenkins):
     def _download(self, destination):
         headers = {}
         headers["User-Agent"] = "BobBuildTool/{}".format(BOB_VERSION)
-        context = None if self.__sslVerify else ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+        context = None if self.__sslVerify else sslNoVerifyContext()
         if os.path.isfile(destination) and self.__url.startswith("http"):
             # Try to avoid download if possible
             headers["If-Modified-Since"] = time2HTTPDate(os.stat(destination).st_mtime)

pym/bob/utils.py

@@ -640,7 +640,6 @@ def __init__(self):
 
         if sys.platform == 'win32':
             loop = asyncio.ProactorEventLoop()
-            asyncio.set_event_loop(loop)
             multiprocessing.set_start_method('spawn')
             executor = concurrent.futures.ProcessPoolExecutor()
         else:
@@ -651,7 +650,7 @@ def __init__(self):
             # have to ignore BrokenProcessPool errors as we will likely hit them.
             # To "prime" the process pool a dummy workload must be executed because
             # the processes are spawned lazily.
-            loop = asyncio.get_event_loop()
+            loop = asyncio.new_event_loop()
             origSigInt = signal.getsignal(signal.SIGINT)
             signal.signal(signal.SIGINT, signal.SIG_IGN)
 
@@ -689,11 +688,15 @@ def run(self):
         self.__executor = executor
 
     def __enter__(self):
+        import asyncio
+        asyncio.set_event_loop(self.__loop)
         return (self.__loop, self.__executor)
 
     def __exit__(self, exc_type, exc_value, traceback):
+        import asyncio
         self.__executor.shutdown()
         self.__loop.close()
+        asyncio.set_event_loop(None)
 
 
 async def run(args, universal_newlines=False, check=False, shell=False, **kwargs):
@@ -730,3 +733,26 @@ async def check_output(args, **kwargs):
     """The subprocess.check_output() call as coroutine."""
     import subprocess
     return (await run(args, check=True, stdout=subprocess.PIPE, **kwargs)).stdout
+
+def runInEventLoop(coro):
+    """Backwards compatibility stub for asyncio.run()"""
+    import asyncio
+
+    if sys.version_info.minor >= 7:
+        return asyncio.run(coro)
+
+    loop = asyncio.new_event_loop()
+    try:
+        asyncio.set_event_loop(loop)
+        return loop.run_until_complete(coro)
+    finally:
+        asyncio.set_event_loop(None)
+        loop.close()
+
+def sslNoVerifyContext():
+    """Generate a SSL context that does not validate certificates."""
+    import ssl
+    context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+    context.check_hostname = False
+    context.verify_mode = ssl.CERT_NONE
+    return context

test/jenkins/jenkins_mock.py

@@ -55,9 +55,9 @@ def do_POST(self):
                 self.end_headers()
                 return
         if 'doDelete' in self.path:
+            self.server.rxJenkinsData((self.path , ''))
             self.send_response(302)
             self.end_headers()
-            self.server.rxJenkinsData((self.path , ''))
             return
         if 'createItem' in self.path:
             length = int(self.headers.get('content-length',0))