Preparing tests to run against development build (#382)

* Preparing tests to run against development build

* Foxx tests

* Foxx tests

Author: apetenchea

.circleci/config.yml

@@ -1,7 +1,14 @@
 version: 2.1
 
-orbs:
-  codecov: codecov/codecov@3.3.0
+executors:
+  python-container:
+    docker:
+      - image: cimg/python:3.12
+    resource_class: small
+  python-vm:
+    machine:
+      image: ubuntu-2404:current
+    resource_class: medium
 
 workflows:
   ci:
@@ -14,98 +21,81 @@ workflows:
               python_version: ["3.10", "3.11", "3.12"]
               arangodb_config: ["single", "cluster"]
               arangodb_license: ["enterprise"]
-              arangodb_version: ["latest"]
+              arangodb_version: ["3.12"]
 
 jobs:
   lint:
-    docker:
-      - image: python:latest
+    executor: python-container
+    resource_class: small
     steps:
       - checkout
       - run:
           name: Install Dependencies
           command: pip install .[dev]
-
       - run:
-          name: Run black
-          command: black --check --verbose --diff --color --config=pyproject.toml ./arango ./tests/
-
+            name: Run black
+            command: black --check --verbose --diff --color --config=pyproject.toml ./arango ./tests/
       - run:
           name: Run flake8
           command: flake8 ./arango ./tests
-
       - run:
           name: Run isort
           command: isort --check ./arango ./tests
-
       - run:
           name: Run mypy
           command: mypy ./arango
-
   test:
     parameters:
-      python_version:
-        type: string
-      arangodb_config:
-        type: string
-      arangodb_license:
-        type: string
-      arangodb_version:
-        type: string
-    # TODO: Reconsider using a docker image instead of a machine
-    # i.e cimg/python:<< parameters.python_version >>
-    machine:
-      image: ubuntu-2204:current
+        python_version:
+            type: string
+        arangodb_config:
+            type: string
+        arangodb_license:
+            type: string
+        arangodb_version:
+            type: string
+    executor: python-vm
     steps:
       - checkout
-
       - run:
-          name: Set Up ArangoDB
+          name: Setup ArangoDB
           command: |
             chmod +x starter.sh
             ./starter.sh << parameters.arangodb_config >> << parameters.arangodb_license >> << parameters.arangodb_version >>
-
       - restore_cache:
           key: pip-and-local-cache
-
-      # TODO: Revisit this bottleneck
       - run:
           name: Setup Python
           command: |
             pyenv --version
             pyenv install -f << parameters.python_version >>
             pyenv global << parameters.python_version >>
-
       - run:
-          name: "Install Dependencies"
+          name: Install Dependencies
           command: pip install -e .[dev]
-
       - run: docker ps -a
-
       - run: docker logs arango
-
       - run:
-          name: "Run pytest"
+          name: Run pytest
           command: |
             mkdir test-results
+            mkdir htmlcov
 
             args=("--junitxml=test-results/junit.xml" "--log-cli-level=DEBUG" "--host" "localhost" "--port=8529")
             if [ << parameters.arangodb_config >> = "cluster" ]; then
               args+=("--cluster" "--port=8539" "--port=8549")
             fi
 
-            if [ << parameters.arangodb_license >> = "enterprise" ]; then
-              args+=("--enterprise")
+            if [ << parameters.arangodb_license >> != "enterprise" ]; then
+              args+=("--skip" "enterprise")
             fi
 
             echo "Running pytest with args: ${args[@]}"
-            pytest --cov=arango --cov-report=xml --cov-report term-missing --color=yes --code-highlight=yes "${args[@]}"
-
+            pytest --cov=arango --cov-report=html:htmlcov --color=yes --code-highlight=yes "${args[@]}"
+      - store_artifacts:
+          path: htmlcov
+          destination: coverage-report
       - store_artifacts:
           path: test-results
-
       - store_test_results:
           path: test-results
-
-      - codecov/upload:
-        file: coverage.xml

docs/contributing.rst

@@ -59,7 +59,7 @@ To run the test suite (use your own host, port and root password):
     ~$ pip install pytest
     ~$ git clone https://github.com/arangodb/python-arango.git
     ~$ cd python-arango
-    ~$ py.test --complete --host=127.0.0.1 --port=8529 --passwd=passwd
+    ~$ pytest --cluster --host=127.0.0.1 --port=8529 --password=passwd
 
 To run the test suite with coverage report:
 
@@ -68,7 +68,7 @@ To run the test suite with coverage report:
     ~$ pip install coverage pytest pytest-cov
     ~$ git clone https://github.com/arangodb/python-arango.git
     ~$ cd python-arango
-    ~$ py.test --complete --host=127.0.0.1 --port=8529 --passwd=passwd --cov=kq
+    ~$ pytest --cluster --host=127.0.0.1 --port=8529 --password=passwd --cov=kq
 
 As the test suite creates real databases and jobs, it should only be run in
 development environments.

pyproject.toml

@@ -57,6 +57,7 @@ dev = [
     "sphinx",
     "sphinx_rtd_theme",
     "types-requests",
+    "allure-pytest>=2.15",
     "types-setuptools",
 ]
 

tests/conftest.py

@@ -39,7 +39,9 @@ class GlobalData:
     cluster: bool = None
     complete: bool = None
     replication: bool = None
-    enterprise: bool = None
+    skip: list[str] = None
+    foxx_path: str = None
+    backup_path: str = None
     secret: str = None
     root_password: str = None
     db_version: version = version.parse("0.0.0")
@@ -49,24 +51,70 @@ class GlobalData:
 
 
 def pytest_addoption(parser):
-    parser.addoption("--host", action="store", default="127.0.0.1")
-    parser.addoption("--port", action="append", default=None)
-    parser.addoption("--passwd", action="store", default="passwd")
-    parser.addoption("--complete", action="store_true")
-    parser.addoption("--cluster", action="store_true")
-    parser.addoption("--replication", action="store_true")
-    parser.addoption("--enterprise", action="store_true")
-    parser.addoption("--secret", action="store", default="secret")
+    parser.addoption(
+        "--host", action="store", default="127.0.0.1", help="ArangoDB host address"
+    )
+    parser.addoption(
+        "--port", action="append", default=None, help="ArangoDB coordinator ports"
+    )
+    parser.addoption(
+        "--root", action="store", default="root", help="ArangoDB root user"
+    )
+    parser.addoption(
+        "--password", action="store", default="passwd", help="ArangoDB password"
+    )
+    parser.addoption(
+        "--secret", action="store", default="secret", help="ArangoDB JWT secret"
+    )
+    parser.addoption(
+        "--cluster", action="store_true", help="Run tests in a cluster setup"
+    )
+    parser.addoption(
+        "--complete",
+        action="store_true",
+        help="Run extra async and transaction tests (not supported)",
+    )
+    parser.addoption("--replication", action="store_true", help="Run replication tests")
+    parser.addoption(
+        "--foxx-path",
+        action="store",
+        default="/tests/static/service.zip",
+        help="Foxx tests service path",
+    )
+    parser.addoption(
+        "--backup-path",
+        action="store",
+        default="local://tmp",
+        help="Backup tests repository path",
+    )
+    parser.addoption(
+        "--skip",
+        action="store",
+        nargs="*",
+        choices=[
+            "backup",  # backup tests
+            "batch",  # batch API tests (deprecated)
+            "jwt-secret-keyfile",  # server was not configured with a keyfile
+            "foxx",  # foxx is not supported
+            "js-transactions",  # javascript transactions are not supported
+            "task",  # tasks API
+            "enterprise",  # skip what used to be "enterprise-only" before 3.12
+        ],
+        default=[],
+        help="Skip specific tests",
+    )
 
 
 def pytest_configure(config):
     ports = config.getoption("port")
     if ports is None:
         ports = ["8529"]
-    hosts = [f"http://{config.getoption('host')}:{p}" for p in ports]
+    hosts = [f"http://{config.getoption('host')}:{p}" for p in ports]  # noqa: E231
     url = hosts[0]
     secret = config.getoption("secret")
     cluster = config.getoption("cluster")
+    root_password = config.getoption("password")
+    root_user = config.getoption("root")
 
     host_resolver = "fallback"
     http_client = DefaultHTTPClient(request_timeout=120)
@@ -76,8 +124,8 @@ def pytest_configure(config):
     )
     sys_db = client.db(
         name="_system",
-        username="root",
-        password=config.getoption("passwd"),
+        username=root_user,
+        password=root_password,
         superuser_token=generate_jwt(secret),
         verify=True,
     )
@@ -148,9 +196,11 @@ def pytest_configure(config):
     global_data.cluster = cluster
     global_data.complete = config.getoption("complete")
     global_data.replication = config.getoption("replication")
-    global_data.enterprise = config.getoption("enterprise")
     global_data.secret = secret
-    global_data.root_password = config.getoption("passwd")
+    global_data.root_password = root_password
+    global_data.skip = config.getoption("skip")
+    global_data.backup_path = config.getoption("backup_path")
+    global_data.foxx_path = config.getoption("foxx_path")
 
 
 # noinspection PyShadowingNames
@@ -186,7 +236,7 @@ def pytest_unconfigure(*_):  # pragma: no cover
             sys_db.delete_collection(col_name, ignore_missing=True)
 
     # # Remove all backups.
-    if global_data.enterprise:
+    if "backup" not in global_data.skip and "enterprise" not in global_data.skip:
         for backup_id in sys_db.backup.get()["list"].keys():
             sys_db.backup.delete(backup_id)
 
@@ -223,16 +273,6 @@ def pytest_generate_tests(metafunc):
             bad_async_db._executor = TestAsyncApiExecutor(bad_conn)
             bad_dbs.append(bad_async_db)
 
-            # Skip test batch databases, as they are deprecated.
-            """
-            tst_batch_db = StandardDatabase(tst_conn)
-            tst_batch_db._executor = TestBatchExecutor(tst_conn)
-            tst_dbs.append(tst_batch_db)
-            bad_batch_bdb = StandardDatabase(bad_conn)
-            bad_batch_bdb._executor = TestBatchExecutor(bad_conn)
-            bad_dbs.append(bad_batch_bdb)
-            """
-
     if "db" in metafunc.fixturenames and "bad_db" in metafunc.fixturenames:
         metafunc.parametrize("db,bad_db", zip(tst_dbs, bad_dbs))
 
@@ -431,11 +471,21 @@ def replication():
     return global_data.replication
 
 
-@pytest.fixture(autouse=False)
-def enterprise():
-    return global_data.enterprise
-
-
 @pytest.fixture(autouse=False)
 def secret():
     return global_data.secret
+
+
+@pytest.fixture
+def backup_path():
+    return global_data.backup_path
+
+
+@pytest.fixture
+def foxx_path():
+    return global_data.foxx_path
+
+
+@pytest.fixture
+def skip_tests():
+    return global_data.skip

tests/static/cluster-3.11.conf

@@ -9,7 +9,7 @@
 from tests.helpers import assert_raises, generate_analyzer_name
 
 
-def test_analyzer_management(db, bad_db, cluster, enterprise, db_version):
+def test_analyzer_management(db, bad_db, skip_tests, db_version):
     analyzer_name = generate_analyzer_name()
     full_analyzer_name = db.name + "::" + analyzer_name
     bad_analyzer_name = generate_analyzer_name()
@@ -70,7 +70,7 @@ def test_analyzer_management(db, bad_db, cluster, enterprise, db_version):
     assert db.delete_analyzer(analyzer_name, ignore_missing=True) is False
 
     # Test create geo_s2 analyzer (EE only)
-    if enterprise:
+    if "enterprise" not in skip_tests:
         analyzer_name = generate_analyzer_name()
         result = db.create_analyzer(analyzer_name, "geo_s2", {})
         assert result["type"] == "geo_s2"

tests/static/setup.sh

@@ -254,8 +254,8 @@ def test_aql_query_management(db_version, db, sys_db, bad_db, col, docs):
     assert err.value.error_code in {11, 1228}
 
 
-def test_aql_query_force_one_shard_attribute_value(db, db_version, enterprise, cluster):
-    if not enterprise or not cluster:
+def test_aql_query_force_one_shard_attribute_value(db, skip_tests, cluster):
+    if "enterprise" in skip_tests or not cluster:
         return
 
     name = generate_col_name()