From ab17dfd52fb5acc6aa023a9cebb65de2b4eb361d Mon Sep 17 00:00:00 2001
From: Christian Hohnstaedt <christian@hohnstaedt.de>
Date: Mon, 15 May 2023 15:39:23 +0200
Subject: [PATCH] Simplify key checking algorithm

Since we dropped OpenSSL < 1.1.1 support, we can use
 EVP_PKEY_public_check() and EVP_PKEY_check()

Thanks to discussion in:
https://github.com/ya-isakov/xca/commit/acb75afa6d33a150ab998e16bbb3159113e141a2
---
 lib/BioByteArray.cpp |  2 ++
 lib/pki_evp.cpp      | 53 ++++++--------------------------------------
 lib/pki_evp.h        |  2 +-
 lib/pki_key.cpp      | 40 ++++-----------------------------
 lib/pki_key.h        |  3 +--
 5 files changed, 15 insertions(+), 85 deletions(-)

diff --git a/lib/BioByteArray.cpp b/lib/BioByteArray.cpp
index b8169685..cfe49099 100644
--- a/lib/BioByteArray.cpp
+++ b/lib/BioByteArray.cpp
@@ -60,6 +60,7 @@ BIO *BioByteArray::bio()
 {
 	if (!read_write) {
 		read_write = BIO_new(BIO_s_mem());
+		Q_CHECK_PTR(read_write);
 		biowrite(store);
 		store.fill(0);
 		store.clear();
@@ -72,6 +73,7 @@ BIO *BioByteArray::ro()
 	if (!read_only)
 		read_only = BIO_new_mem_buf(
 			(void*)store.constData(), store.length());
+	Q_CHECK_PTR(read_only);
 	return read_only;
 }
 
diff --git a/lib/pki_evp.cpp b/lib/pki_evp.cpp
index 76a3865e..9535c2bf 100644
--- a/lib/pki_evp.cpp
+++ b/lib/pki_evp.cpp
@@ -139,6 +139,7 @@ void pki_evp::generate(const keyjob &task)
 	case EVP_PKEY_ED25519: {
 		EVP_PKEY *pkey = NULL;
 		EVP_PKEY_CTX *pctx = EVP_PKEY_CTX_new_id(EVP_PKEY_ED25519, NULL);
+		Q_CHECK_PTR(pctx);
 		EVP_PKEY_keygen_init(pctx);
 		EVP_PKEY_keygen(pctx, &pkey);
 		EVP_PKEY_CTX_free(pctx);
@@ -810,56 +811,16 @@ void pki_evp::writeKey(XFile &file, const EVP_CIPHER *enc,
 	file.write(b);
 }
 
-bool pki_evp::verify_priv(EVP_PKEY *pkey) const
+bool pki_evp::verify(EVP_PKEY *pkey) const
 {
-	bool verify = true;
-	unsigned char data[32], sig[1024];
-	size_t datalen = sizeof data, siglen = sizeof sig;
-	EVP_MD_CTX *ctx = NULL;
-	const EVP_MD *md = EVP_sha256();
-	EVP_PKEY_CTX *pkctx = NULL;
-
 	if (!EVP_PKEY_isPrivKey(pkey))
-		return true;
-	do {
-		ctx = EVP_MD_CTX_new();
-		pki_ign_openssl_error();
-		RAND_bytes(data, datalen);
-		Q_CHECK_PTR(ctx);
-		verify = false;
+		return pki_key::verify(pkey);
 
-		/* Sign some random data in "data" */
-#ifdef EVP_PKEY_ED25519
-		if (EVP_PKEY_id(pkey) == EVP_PKEY_ED25519)
-			md = NULL;
-#endif
-		if (!EVP_DigestSignInit(ctx, &pkctx, md, NULL, pkey))
-			break;
+	EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new(pkey, NULL);
+	Q_CHECK_PTR(ctx);
+	bool verify = EVP_PKEY_check(ctx);
+	EVP_PKEY_CTX_free(ctx);
 
-		if (EVP_PKEY_id(pkey) == EVP_PKEY_RSA)
-			EVP_PKEY_CTX_set_rsa_padding(pkctx, RSA_PKCS1_PADDING);
-
-		if (!EVP_DigestSign(ctx, sig, &siglen, data, datalen))
-			break;
-
-		/* Verify the signature */
-		if (!EVP_DigestVerifyInit(ctx, NULL, md, NULL, pkey))
-			break;
-
-		if (EVP_DigestVerify(ctx, sig, siglen, data, datalen) != 1)
-			break;
-
-		verify = true;
-	} while (0);
-
-	if (ctx)
-		EVP_MD_CTX_free(ctx);
-
-	if (EVP_PKEY_id(pkey) == EVP_PKEY_RSA && EVP_PKEY_isPrivKey(pkey)) {
-		const RSA *rsa = EVP_PKEY_get0_RSA(pkey);
-		if (RSA_check_key(rsa) != 1)
-			verify = false;
-	}
 	pki_openssl_error();
 	return verify;
 }
diff --git a/lib/pki_evp.h b/lib/pki_evp.h
index 4b904b9e..53b97a4c 100644
--- a/lib/pki_evp.h
+++ b/lib/pki_evp.h
@@ -66,7 +66,7 @@ class pki_evp: public pki_key
 		void writePKCS8(XFile &file, const EVP_CIPHER *enc,
 				pem_password_cb *cb, bool pem) const;
 		void writePVKprivate(XFile &file, pem_password_cb *cb) const;
-		bool verify_priv(EVP_PKEY *pkey) const;
+		bool verify(EVP_PKEY *pkey) const;
 		QVariant getIcon(const dbheader *hd) const;
 		bool sqlUpdatePrivateKey();
 		QSqlError insertSqlData();
diff --git a/lib/pki_key.cpp b/lib/pki_key.cpp
index e3c82ddc..9c38d6ce 100644
--- a/lib/pki_key.cpp
+++ b/lib/pki_key.cpp
@@ -789,47 +789,15 @@ void pki_key::writeSSH2public(XFile &file) const
 
 bool pki_key::verify(EVP_PKEY *pkey) const
 {
-	bool verify = true;
-	const BIGNUM *a = NULL;
-	const BIGNUM *b = NULL;
-	const BIGNUM *c = NULL;
+	EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new(pkey, NULL);
+	Q_CHECK_PTR(ctx);
+	bool verify = EVP_PKEY_public_check(ctx);
+	EVP_PKEY_CTX_free(ctx);
 
-	switch (EVP_PKEY_type(EVP_PKEY_id(pkey))) {
-	case EVP_PKEY_RSA:
-		RSA_get0_key(EVP_PKEY_get0_RSA(pkey), &a, &b, NULL);
-		verify = a && b;
-		break;
-	case EVP_PKEY_DSA:
-		DSA_get0_pqg(EVP_PKEY_get0_DSA(pkey), &a, &b, &c);
-		verify = a && b && c;
-		break;
-#ifndef OPENSSL_NO_EC
-	case EVP_PKEY_EC:
-		verify = EC_KEY_check_key(EVP_PKEY_get0_EC_KEY(pkey)) == 1;
-		break;
-#ifdef EVP_PKEY_ED25519
-	case EVP_PKEY_ED25519: {
-		size_t len;
-		verify = EVP_PKEY_get_raw_private_key(pkey, NULL, &len) == 1 &&
-				len == ED25519_KEYLEN;
-		break;
-	}
-#endif
-#endif
-	default:
-		verify = false;
-	}
-	if (verify)
-		verify = verify_priv(pkey);
 	pki_openssl_error();
 	return verify;
 }
 
-bool pki_key::verify_priv(EVP_PKEY *) const
-{
-	return true;
-}
-
 QString pki_key::fingerprint(const QString &format) const
 {
 	const EVP_MD *md;
diff --git a/lib/pki_key.h b/lib/pki_key.h
index c5358023..22bcd813 100644
--- a/lib/pki_key.h
+++ b/lib/pki_key.h
@@ -210,8 +210,7 @@ class pki_key: public pki_base
 		bool compare(const pki_base *ref) const;
 		int getKeyType() const;
 		bool isPrivKey() const;
-		bool verify(EVP_PKEY *pkey) const;
-		virtual bool verify_priv(EVP_PKEY *pkey) const;
+		virtual bool verify(EVP_PKEY *pkey) const;
 		int getUcount() const;
 		void setUcount(int c)
 		{