pkey: EVP_DigestVerify doesn't return -1 in AWS-LC

EVP_DigestVerify in OpenSSL returns 0 to indicate a signature
verification failure and can return -1  to indicate other
failures, such as invalid ASN1 contents. ruby/openssl also
reflects that by returning false with 0 and raising an error
with -1.
EVP_DigestVerify in AWS-LC simply returns 0 for any failure.

Author: samuel40791765

test/openssl/test_pkey_dsa.rb

@@ -92,19 +92,19 @@ def test_sign_verify_raw
     sig = key.syssign(digest)
     assert_equal true, key.sysverify(digest, sig)
     assert_equal false, key.sysverify(digest, invalid_sig)
-    assert_raise(OpenSSL::PKey::DSAError) { key.sysverify(digest, malformed_sig) }
+    assert_sign_verify_false_or_error{ key.sysverify(digest, malformed_sig) }
     assert_equal true, key.verify_raw(nil, sig, digest)
     assert_equal false, key.verify_raw(nil, invalid_sig, digest)
-    assert_raise(OpenSSL::PKey::PKeyError) { key.verify_raw(nil, malformed_sig, digest) }
+    assert_sign_verify_false_or_error { key.verify_raw(nil, malformed_sig, digest) }
 
     # Sign by #sign_raw
     sig = key.sign_raw(nil, digest)
     assert_equal true, key.sysverify(digest, sig)
     assert_equal false, key.sysverify(digest, invalid_sig)
-    assert_raise(OpenSSL::PKey::DSAError) { key.sysverify(digest, malformed_sig) }
+    assert_sign_verify_false_or_error { key.sysverify(digest, malformed_sig) }
     assert_equal true, key.verify_raw(nil, sig, digest)
     assert_equal false, key.verify_raw(nil, invalid_sig, digest)
-    assert_raise(OpenSSL::PKey::PKeyError) { key.verify_raw(nil, malformed_sig, digest) }
+    assert_sign_verify_false_or_error { key.verify_raw(nil, malformed_sig, digest) }
   end
 
   def test_DSAPrivateKey

test/openssl/test_pkey_ec.rb

@@ -152,19 +152,19 @@ def test_sign_verify_raw
     sig = key.dsa_sign_asn1(data1)
     assert_equal true, key.dsa_verify_asn1(data1, sig)
     assert_equal false, key.dsa_verify_asn1(data2, sig)
-    assert_raise(OpenSSL::PKey::ECError) { key.dsa_verify_asn1(data1, malformed_sig) }
+    assert_sign_verify_false_or_error { key.dsa_verify_asn1(data1, malformed_sig) }
     assert_equal true, key.verify_raw(nil, sig, data1)
     assert_equal false, key.verify_raw(nil, sig, data2)
-    assert_raise(OpenSSL::PKey::PKeyError) { key.verify_raw(nil, malformed_sig, data1) }
+    assert_sign_verify_false_or_error { key.verify_raw(nil, malformed_sig, data1) }
 
     # Sign by #sign_raw
     sig = key.sign_raw(nil, data1)
     assert_equal true, key.dsa_verify_asn1(data1, sig)
     assert_equal false, key.dsa_verify_asn1(data2, sig)
-    assert_raise(OpenSSL::PKey::ECError) { key.dsa_verify_asn1(data1, malformed_sig) }
+    assert_sign_verify_false_or_error { key.dsa_verify_asn1(data1, malformed_sig) }
     assert_equal true, key.verify_raw(nil, sig, data1)
     assert_equal false, key.verify_raw(nil, sig, data2)
-    assert_raise(OpenSSL::PKey::PKeyError) { key.verify_raw(nil, malformed_sig, data1) }
+    assert_sign_verify_false_or_error{ key.verify_raw(nil, malformed_sig, data1) }
   end
 
   def test_dsa_sign_asn1_FIPS186_3

test/openssl/utils.rb

@@ -286,6 +286,14 @@ def check_component(base, test, keys)
       assert_equal base.send(comp), test.send(comp)
     }
   end
+
+  def assert_sign_verify_false_or_error
+    ret = yield
+  rescue => e
+    assert_kind_of(OpenSSL::PKey::PKeyError, e)
+  else
+    assert_equal(false, ret)
+  end
 end
 
 module OpenSSL::Certs