modsign: Make sign-file determine the format of the X.509 cert

Make sign-file determine the format of the X.509 certificate by reading the
first two bytes and seeing if the first byte is 0x30 and the second
0x81-0x84.  If this is the case, assume it's DER encoded, otherwise assume
it to be PEM encoded.

Without this, it gets awkward to deal with the error messages from
d2i_X509_bio() when we want to call BIO_reset() and then PEM_read_bio() in
case the certificate was PEM encoded rather than X.509 encoded.

Reported-by: Ben Hutchings <ben@decadent.org.uk>
Signed-off-by: David Howells <dhowells@redhat.com>
Tested-by: Ben Hutchings <ben@decadent.org.uk>
cc: David Woodhouse <dwmw2@infradead.org>
cc: Juerg Haefliger <juerg.haefliger@hpe.com>
cc: Ben Hutchings <ben@decadent.org.uk>

diff --git a/scripts/sign-file.c b/scripts/sign-file.c
index d912d5a56a5ebf05f3e1f2fb0f93cfd028907167..53af6dc3e6c1779fbde3c5a641a880f52aa768e5 100755 (executable)
--- a/scripts/sign-file.c
+++ b/scripts/sign-file.c
@@ -1,6 +1,6 @@
 /* Sign a module file using the given key.
  *
- * Copyright © 2014-2015 Red Hat, Inc. All Rights Reserved.
+ * Copyright © 2014-2016 Red Hat, Inc. All Rights Reserved.
  * Copyright © 2015      Intel Corporation.
  * Copyright © 2016      Hewlett Packard Enterprise Development LP
  *
@@ -167,19 +167,37 @@ static EVP_PKEY *read_private_key(const char *private_key_name)
 
 static X509 *read_x509(const char *x509_name)
 {
+       unsigned char buf[2];
        X509 *x509;
        BIO *b;
+       int n;
 
        b = BIO_new_file(x509_name, "rb");
        ERR(!b, "%s", x509_name);
-       x509 = d2i_X509_bio(b, NULL); /* Binary encoded X.509 */
-       if (!x509) {
-               ERR(BIO_reset(b) != 1, "%s", x509_name);
-               x509 = PEM_read_bio_X509(b, NULL, NULL,
-                                        NULL); /* PEM encoded X.509 */
-               if (x509)
-                       drain_openssl_errors();
+
+       /* Look at the first two bytes of the file to determine the encoding */
+       n = BIO_read(b, buf, 2);
+       if (n != 2) {
+               if (BIO_should_retry(b)) {
+                       fprintf(stderr, "%s: Read wanted retry\n", x509_name);
+                       exit(1);
+               }
+               if (n >= 0) {
+                       fprintf(stderr, "%s: Short read\n", x509_name);
+                       exit(1);
+               }
+               ERR(1, "%s", x509_name);
        }
+
+       ERR(BIO_reset(b) != 0, "%s", x509_name);
+
+       if (buf[0] == 0x30 && buf[1] >= 0x81 && buf[1] <= 0x84)
+               /* Assume raw DER encoded X.509 */
+               x509 = d2i_X509_bio(b, NULL);
+       else
+               /* Assume PEM encoded X.509 */
+               x509 = PEM_read_bio_X509(b, NULL, NULL, NULL);
+
        BIO_free(b);
        ERR(!x509, "%s", x509_name);