x509name: fix OpenSSL::X509::Name#{cmp,<=>}

Fix wrong use of X509_NAME_cmp() return value. OpenSSL::X509::Name#<=>
could return 0 when the two objects aren't identical.

Reported by Tyler Eckstein. CVE-2018-16395.

Reference: https://hackerone.com/reports/387250

Author: rhenium

ext/openssl/ossl_x509name.c

@@ -358,7 +358,7 @@ ossl_x509name_cmp(VALUE self, VALUE other)
 
     result = ossl_x509name_cmp0(self, other);
     if (result < 0) return INT2FIX(-1);
-    if (result > 1) return INT2FIX(1);
+    if (result > 0) return INT2FIX(1);
 
     return INT2FIX(0);
 }

test/test_x509name.rb

@@ -330,10 +330,16 @@ def test_equals2
   end
 
   def test_spaceship
-    n1 = OpenSSL::X509::Name.parse 'CN=a'
-    n2 = OpenSSL::X509::Name.parse 'CN=b'
-
-    assert_equal(-1, n1 <=> n2)
+    n1 = OpenSSL::X509::Name.new([["CN", "a"]])
+    n2 = OpenSSL::X509::Name.new([["CN", "a"]])
+    n3 = OpenSSL::X509::Name.new([["CN", "ab"]])
+
+    assert_equal 0, n1 <=> n2
+    assert_equal -1, n1 <=> n3
+    assert_equal 0, n2 <=> n1
+    assert_equal -1, n2 <=> n3
+    assert_equal 1, n3 <=> n1
+    assert_equal 1, n3 <=> n2
   end
 
   def name_hash(name)