MEN-3730 Switch all client crypto operations over to OpenSSL.

All server communication goes through openssl:

* custom dial function is provided
* new verification errors are handled

Unit tests fixes:

* new certs
* new call startTestHTTPS, to avoid build-in localhostCert from httptest
* new tests:
  * TestClientAuthDepthZeroSelfSignedCert
    X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT
  * TestClientAuthEndEntityKeyTooSmall
    X509_V_ERR_EE_KEY_TOO_SMALL
* test host added to /etc/hosts in .gitlab-ci.yml
* added host verification error test
* added no verify test

ChangeLog:Switch to OpenSSL for all server communication.
Signed-off-by: Peter Grzybowski <peter@northern.tech>

Author: merlin-northern

client/client.go

@@ -26,6 +26,7 @@ import (
 	"strings"
 	"time"
 
+	openssl "github.com/Linutronix/golang-openssl"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/net/http2"
@@ -280,6 +281,49 @@ func newHttpClient() *http.Client {
 	return &http.Client{}
 }
 
+func dialOpenSSL(conf Config, network string, addr string) (net.Conn, error) {
+	contextSSL, err := openssl.NewCtx()
+	// probably should consider reusing the context, but then we
+	// have to propagate it with every request.
+	err = contextSSL.LoadVerifyLocations(conf.ServerCert, "")
+	if err != nil {
+		return nil, err
+	}
+	flags := openssl.DialFlags(0)
+
+	if conf.NoVerify {
+		flags = openssl.InsecureSkipHostVerification
+	}
+
+	conn, err := openssl.Dial("tcp", addr, contextSSL, flags)
+	if err != nil {
+		return nil, err
+	}
+
+	v := conn.VerifyResult()
+	if v != openssl.Ok {
+		if v == openssl.CertHasExpired {
+			return nil, errors.Errorf("certificate has expired, "+
+				"openssl verify rc: %d server cert file: %s", v, conf.ServerCert)
+		}
+		if v == openssl.DepthZeroSelfSignedCert {
+			return nil, errors.Errorf("depth zero self-signed certificate, "+
+				"openssl verify rc: %d server cert file: %s", v, conf.ServerCert)
+		}
+		if v == 0x42 { //X509_V_ERR_EE_KEY_TOO_SMALL
+			return nil, errors.Errorf("end entity key too short, "+
+				"openssl verify rc: %d server cert file: %s", v, conf.ServerCert)
+		}
+		if v == 0x14 { //X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY
+			return nil, errors.Errorf("certificate signed by unknown authority, "+
+				"openssl verify rc: %d server cert file: %s", v, conf.ServerCert)
+		}
+		return nil, errors.Errorf("not a valid certificate, "+
+			"openssl verify rc: %d server cert file: %s", v, conf.ServerCert)
+	}
+	return conn, err
+}
+
 func newHttpsClient(conf Config) (*http.Client, error) {
 	client := newHttpClient()
 
@@ -295,6 +339,9 @@ func newHttpsClient(conf Config) (*http.Client, error) {
 	transport := http.Transport{
 		TLSClientConfig: &tlsc,
 		Proxy:           http.ProxyFromEnvironment,
+		DialTLS: func(network string, addr string) (net.Conn, error) {
+			return dialOpenSSL(conf, network, addr)
+		},
 	}
 
 	client.Transport = &transport

client/client_auth_test.go

@@ -19,7 +19,6 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
-	"net/http/httptest"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -88,13 +87,15 @@ func TestClientAuth(t *testing.T) {
 		http.Header{},
 	}
 
-	ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	ts := startTestHTTPS(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		responder.headers = r.Header
 		w.WriteHeader(responder.httpStatus)
 		w.Header().Set("Content-Type", "application/json")
 
 		fmt.Fprint(w, responder.data)
-	}))
+	}),
+		localhostCert,
+		localhostKey)
 	defer ts.Close()
 
 	ac, err := NewApiClient(
@@ -122,8 +123,10 @@ func TestClientAuth(t *testing.T) {
 }
 
 func TestClientAuthExpiredCert(t *testing.T) {
-	ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-	}))
+	ts := startTestHTTPS(
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}),
+		localhostCertExpired,
+		localhostKeyExpired)
 	defer ts.Close()
 
 	ac, err := NewApiClient(
@@ -145,8 +148,10 @@ func TestClientAuthExpiredCert(t *testing.T) {
 }
 
 func TestClientAuthUnknownAuthorityCert(t *testing.T) {
-	ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-	}))
+	ts := startTestHTTPS(
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}),
+		localhostCertUnknown,
+		localhostKeyUnknown)
 	defer ts.Close()
 
 	ac, err := NewApiClient(
@@ -164,12 +169,70 @@ func TestClientAuthUnknownAuthorityCert(t *testing.T) {
 	rsp, err := client.Request(ac, ts.URL, msger)
 	assert.Error(t, err)
 	assert.Contains(t, err.Error(), "certificate signed by unknown authority")
+	// see https://github.com/openssl/openssl/blob/OpenSSL_1_1_1-stable/crypto/x509/x509_vfy.c#L3268
+	//     for self-signed openssl always returns self-signed error; either
+	//     X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT or X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN
+	//     and never X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT or X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY
+	assert.Nil(t, rsp)
+}
+
+//X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT
+func TestClientAuthDepthZeroSelfSignedCert(t *testing.T) {
+	ts := startTestHTTPS(
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}),
+		localhostCert,
+		localhostKey)
+	defer ts.Close()
+
+	ac, err := NewApiClient(
+		Config{"server.zero.depth.self.signed.crt", true, false},
+	)
+	assert.NotNil(t, ac)
+	assert.NoError(t, err)
+
+	client := NewAuth()
+	assert.NotNil(t, client)
+
+	msger := &testAuthDataMessenger{
+		reqData: []byte("foobar"),
+	}
+	rsp, err := client.Request(ac, ts.URL, msger)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "depth zero self-signed certificate")
+	assert.Nil(t, rsp)
+}
+
+//X509_V_ERR_EE_KEY_TOO_SMALL
+func TestClientAuthEndEntityKeyTooSmall(t *testing.T) {
+	ts := startTestHTTPS(
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}),
+		localhostCertShortEEKey,
+		localhostKeyShortEEKey)
+	defer ts.Close()
+
+	ac, err := NewApiClient(
+		Config{"server.crt", true, false},
+	)
+	assert.NotNil(t, ac)
+	assert.NoError(t, err)
+
+	client := NewAuth()
+	assert.NotNil(t, client)
+
+	msger := &testAuthDataMessenger{
+		reqData: []byte("foobar"),
+	}
+	rsp, err := client.Request(ac, ts.URL, msger)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "end entity key too short")
 	assert.Nil(t, rsp)
 }
 
 func TestClientAuthNoCert(t *testing.T) {
-	ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-	}))
+	ts := startTestHTTPS(
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}),
+		localhostCert,
+		localhostKey)
 	defer ts.Close()
 
 	ac, err := NewApiClient(
@@ -178,3 +241,79 @@ func TestClientAuthNoCert(t *testing.T) {
 	assert.NotNil(t, ac)
 	assert.NoError(t, err)
 }
+
+func TestClientAuthHostValidationNocheck(t *testing.T) {
+	ts := startTestHTTPS(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}),
+		localhostCert,
+		localhostKey)
+	defer ts.Close()
+
+	ac, err := NewApiClient(
+		Config{"server.crt", true, true},
+	)
+	assert.NotNil(t, ac)
+	assert.NoError(t, err)
+
+	client := NewAuth()
+	assert.NotNil(t, client)
+
+	msger := &testAuthDataMessenger{
+		reqData: []byte("foobar"),
+	}
+	rsp, err := client.Request(ac, ts.URL, msger)
+
+	assert.NoError(t, err)
+	assert.NotNil(t, rsp)
+}
+
+func TestClientAuthHostValidationError(t *testing.T) {
+	ts := startTestHTTPS(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}),
+		localhostCertWrongHost,
+		localhostKeyWrongHost)
+	defer ts.Close()
+
+	ac, err := NewApiClient(
+		Config{"server.unknown-authority.crt", true, false},
+	)
+	assert.NotNil(t, ac)
+	assert.NoError(t, err)
+
+	client := NewAuth()
+	assert.NotNil(t, client)
+
+	msger := &testAuthDataMessenger{
+		reqData: []byte("foobar"),
+	}
+	rsp, err := client.Request(ac, ts.URL, msger)
+
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "Host validation error")
+	assert.Nil(t, rsp)
+}
+
+// this tests for the error that is handled by default 'not a valid certificate'
+// via X509_V_ERR_CA_KEY_TOO_SMALL
+func TestClientAuthNotValidCertificate(t *testing.T) {
+	ts := startTestHTTPS(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}),
+		localhostCertCAKeyTooShort,
+		localhostKeyCAKeyTooShort)
+	defer ts.Close()
+
+	ac, err := NewApiClient(
+		Config{"server.ca.key.too.small.crt", true, false},
+	)
+	assert.NotNil(t, ac)
+	assert.NoError(t, err)
+
+	client := NewAuth()
+	assert.NotNil(t, client)
+
+	msger := &testAuthDataMessenger{
+		reqData: []byte("foobar"),
+	}
+	rsp, err := client.Request(ac, ts.URL, msger)
+
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "not a valid certificate")
+	assert.Nil(t, rsp)
+}

client/client_inventory_test.go

@@ -1,4 +1,4 @@
-// Copyright 2017 Northern.tech AS
+// Copyright 2020 Northern.tech AS
 //
 //    Licensed under the Apache License, Version 2.0 (the "License");
 //    you may not use this file except in compliance with the License.
@@ -16,7 +16,6 @@ package client
 import (
 	"io/ioutil"
 	"net/http"
-	"net/http/httptest"
 	"testing"
 
 	"github.com/pkg/errors"
@@ -35,12 +34,15 @@ func TestInventoryClient(t *testing.T) {
 	}
 
 	// Test server that always responds with 200 code, and specific payload
-	ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(responder.httpStatus)
+	ts := startTestHTTPS(
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(responder.httpStatus)
 
-		responder.recdata, _ = ioutil.ReadAll(r.Body)
-		responder.path = r.URL.Path
-	}))
+			responder.recdata, _ = ioutil.ReadAll(r.Body)
+			responder.path = r.URL.Path
+		}),
+		localhostCert,
+		localhostKey)
 	defer ts.Close()
 
 	ac, err := NewApiClient(

client/client_log_test.go

@@ -1,4 +1,4 @@
-// Copyright 2017 Northern.tech AS
+// Copyright 2020 Northern.tech AS
 //
 //    Licensed under the Apache License, Version 2.0 (the "License");
 //    you may not use this file except in compliance with the License.
@@ -16,7 +16,6 @@ package client
 import (
 	"io/ioutil"
 	"net/http"
-	"net/http/httptest"
 	"testing"
 
 	"github.com/pkg/errors"
@@ -35,12 +34,15 @@ func TestLogUploadClient(t *testing.T) {
 	}
 
 	// Test server that always responds with 200 code, and specific payload
-	ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(responder.httpStatus)
+	ts := startTestHTTPS(
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(responder.httpStatus)
 
-		responder.recdata, _ = ioutil.ReadAll(r.Body)
-		responder.path = r.URL.Path
-	}))
+			responder.recdata, _ = ioutil.ReadAll(r.Body)
+			responder.path = r.URL.Path
+		}),
+		localhostCert,
+		localhostKey)
 	defer ts.Close()
 
 	ac, err := NewApiClient(

client/client_status_test.go

@@ -1,4 +1,4 @@
-// Copyright 2018 Northern.tech AS
+// Copyright 2020 Northern.tech AS
 //
 //    Licensed under the Apache License, Version 2.0 (the "License");
 //    you may not use this file except in compliance with the License.
@@ -16,7 +16,6 @@ package client
 import (
 	"io/ioutil"
 	"net/http"
-	"net/http/httptest"
 	"testing"
 
 	"github.com/pkg/errors"
@@ -35,12 +34,15 @@ func TestStatusClient(t *testing.T) {
 	}
 
 	// Test server that always responds with 200 code, and specific payload
-	ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(responder.httpStatus)
+	ts := startTestHTTPS(
+		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(responder.httpStatus)
 
-		responder.recdata, _ = ioutil.ReadAll(r.Body)
-		responder.path = r.URL.Path
-	}))
+			responder.recdata, _ = ioutil.ReadAll(r.Body)
+			responder.path = r.URL.Path
+		}),
+		localhostCert,
+		localhostKey)
 	defer ts.Close()
 
 	ac, err := NewApiClient(