From ee297eb605de1f6794cdb20fdd911e15a2866599 Mon Sep 17 00:00:00 2001
From: Andrew Gunnerson <chillermillerlong@hotmail.com>
Date: Mon, 24 Oct 2022 10:33:43 -0400
Subject: [PATCH] avbroot/openssl.py: Explicitly set -inform when querying RSA
 modulus

openssl 1.1 does not support autodetecting whether the input file is PEM
or DER encoded.

Issue: #2

Signed-off-by: Andrew Gunnerson <chillermillerlong@hotmail.com>
---
 avbroot/openssl.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/avbroot/openssl.py b/avbroot/openssl.py
index e092857..a2e2268 100644
--- a/avbroot/openssl.py
+++ b/avbroot/openssl.py
@@ -11,10 +11,15 @@ def _get_modulus(path, is_x509):
     certificate.
     '''
 
+    # openssl 1.1 does not support autodetection
+    with open(path, 'rb') as f:
+        is_pem = f.read(1) == b'-'
+
     output = subprocess.check_output([
         'openssl',
         'x509' if is_x509 else 'rsa',
         '-in', path,
+        '-inform', 'PEM' if is_pem else 'DER',
         '-noout',
         '-modulus',
     ])