Support secure transport and certificate-based authentication. (#42)

Signed-off-by: Tao He <linzhu.ht@alibaba-inc.com>

.github/workflows/build-test.yml

@@ -39,6 +39,7 @@ jobs:
                               libssl-dev \
                               libz-dev \
                               lsb-release \
+                              openssl \
                               screenfetch \
                               wget
 
@@ -136,7 +137,7 @@ jobs:
                    -DBUILD_ETCD_TESTS=ON \
                    -DCMAKE_C_COMPILER_LAUNCHER=ccache \
                    -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
-          make -j2
+          make -j`nproc`
           sudo make install
 
       - name: Setup tmate session
@@ -150,16 +151,31 @@ jobs:
           # use etcd v3 api
           export ETCDCTL_API="3"
 
-          cd build
+          rm -rf default.etcd
           /usr/local/bin/etcd &
 
-          # tests without auth
+          sleep 5
-          ./bin/EtcdSyncTest
-          ./bin/EtcdTest
-          ./bin/LockTest
-          ./bin/WatcherTest
 
-          # tests with auth
+          # tests without auth
+          ./build/bin/EtcdSyncTest
+          ./build/bin/EtcdTest
+          ./build/bin/LockTest
+          ./build/bin/WatcherTest
+
+          killall -TERM etcd
+          sleep 5
+
+      - name: Authentication Test
+        run: |
+          export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
+
+          # use etcd v3 api
+          export ETCDCTL_API="3"
+
+          rm -rf default.etcd
+          /usr/local/bin/etcd &
+
+          sleep 5
 
           # for etcd v3.2, v3.3
           if [[ "${{ matrix.etcd }}" == v3.2* ]] || [[ "${{ matrix.etcd }}" == v3.3* ]];
@@ -174,7 +190,7 @@ jobs:
 
           /usr/local/bin/etcdctl auth enable || true
 
-          ./bin/AuthTest
+          ./build/bin/AuthTest
 
           # for etcd v3.2
           if [[ "${{ matrix.etcd }}" == v3.2* ]] || [[ "${{ matrix.etcd }}" == v3.3* ]];
@@ -187,6 +203,36 @@ jobs:
               /usr/local/bin/etcdctl auth disable --user="root" --password="root" || true
           fi
 
+          killall -TERM etcd
+          sleep 5
+
+      - name: Transport Security and Authentication Test
+        run: |
+          export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
+
+          # use etcd v3 api
+          export ETCDCTL_API="3"
+
+          # generate CA certificates
+          ./security-config/reset-ca.sh
+          ./security-config/setup-ca.sh
+
+          rm -rf default.etcd
+          /usr/local/bin/etcd \
+            --cert-file security-config/certs/etcd0.example.com.crt \
+            --key-file security-config/private/etcd0.example.com.key \
+            --client-cert-auth \
+            --trusted-ca-file security-config/certs/ca.crt \
+            --advertise-client-urls=https://127.0.0.1:2379 \
+            --listen-client-urls=https://127.0.0.1:2379 &
+
+          sleep 5
+
+          ./build/bin/SecurityChannelTest
+
+          killall -TERM etcd
+          sleep 5
+
       - name: Check ccache
         run: |
           ccache --show-stats

README.md

@@ -160,7 +160,9 @@ be used by default.
 
 ### Etcd authentication
 
-Etcd [v3's authentication](https://etcd.io/docs/v3.4.0/learning/design-auth-v3/) is currently
+#### v3 authentication
+
+Etcd [v3 authentication](https://etcd.io/docs/v3.4.0/learning/design-auth-v3/) has been
 supported. The `Client::Client` could accept a `username` and `password` as arguments and handle
 the authentication properly.
 
@@ -168,6 +170,13 @@ the authentication properly.
   etcd::Client etcd("http://127.0.0.1:2379", "root", "root");
 ```
 
+Or the etcd client can be constructed explictly:
+
+```c++
+  etcd::Client *etcd = etcd::Client::WithUser(
+                    "http://127.0.0.1:2379", "root", "root");
+```
+
 Enabling v3 authentication requires a bit more work for older versions etcd (etcd 3.2.x and etcd 3.3.x).
 First you need to set the `ETCDCTL_API=3`, then
 
@@ -189,6 +198,43 @@ printf 'root\nroot\n' | /usr/local/bin/etcdctl user add root
 /usr/local/bin/etcdctl --user="root:root" auth disable
 ```
 
+#### transport security
+
+Etcd [transport security](https://etcd.io/docs/v3.4.0/op-guide/security/) and certificate based
+authentication have been supported as well. The `Client::Client` could accept arguments `ca` ,
+`cert` and `key` for CA cert, cert and private key files for the SSL/TLS transport and authentication.
+Note that the later arguments `cert` and `key` could be empty strings or omitted if you just need
+secure transport and don't enable certificate-based client authentication (using the `--client-cert-auth`
+arguments when launching etcd server).
+
+```c++
+  etcd::Client etcd("https://127.0.0.1:2379",
+                    "example.rootca.cert", "example.cert", "example.key",
+                    "round_robin");
+```
+
+Or the etcd client can be constructed explictly:
+
+```c++
+  etcd::Client *etcd = etcd::Client::WithUser(
+                    "https://127.0.0.1:2379",
+                    "example.rootca.cert", "example.cert", "example.key");
+```
+
+Using secure transport but not certificated-based client authentication:
+
+```c++
+  etcd::Client *etcd = etcd::Client::WithUser(
+                    "https://127.0.0.1:2379", "example.rootca.cert");
+```
+
+For more details about setup about security communication between etcd server and client, please
+refer to [transport security](https://etcd.io/docs/v3.4.0/op-guide/security/) in etcd documentation
+and [an example](https://github.com/kelseyhightower/etcd-production-setup) about setup etcd with
+transport security using openssl.
+
+We also provide a tool [`setup-ca.sh`](./security-config/setup-ca.sh) as a helper for development and testing.
+
 ### Reading a value
 
 You can read a value with the ```get``` method of the clinent instance. The only parameter is the

etcd/Client.hpp

@@ -50,6 +50,52 @@ namespace etcd
            std::string const & password,
            std::string const & load_balancer = "round_robin");
 
+    /**
+     * Constructs an etcd client object.
+     *
+     * @param etcd_url is the url of the etcd server to connect to, like "http://127.0.0.1:2379",
+     *                 or multiple url, seperated by ',' or ';'.
+     * @param username username of etcd auth
+     * @param password password of etcd auth
+     * @param load_balancer is the load balance strategy, can be one of round_robin/pick_first/grpclb/xds.
+     */
+    static etcd::Client *WithUser(std::string const & etcd_url,
+           std::string const & username,
+           std::string const & password,
+           std::string const & load_balancer = "round_robin");
+
+    /**
+     * Constructs an etcd client object.
+     *
+     * @param etcd_url is the url of the etcd server to connect to, like "http://127.0.0.1:2379",
+     *                 or multiple url, seperated by ',' or ';'.
+     * @param ca   root CA file for SSL/TLS connection.
+     * @param cert cert chain file for SSL/TLS authentication, could be empty string.
+     * @param key  private key file for SSL/TLS authentication, could be empty string.
+     * @param load_balancer is the load balance strategy, can be one of round_robin/pick_first/grpclb/xds.
+     */
+    Client(std::string const & etcd_url,
+           std::string const & ca,
+           std::string const & cert,
+           std::string const & key,
+           std::string const & load_balancer);
+
+    /**
+     * Constructs an etcd client object.
+     *
+     * @param etcd_url is the url of the etcd server to connect to, like "http://127.0.0.1:2379",
+     *                 or multiple url, seperated by ',' or ';'.
+     * @param ca   root CA file for SSL/TLS connection.
+     * @param cert cert chain file for SSL/TLS authentication, could be empty string.
+     * @param key  private key file for SSL/TLS authentication, could be empty string.
+     * @param load_balancer is the load balance strategy, can be one of round_robin/pick_first/grpclb/xds.
+     */
+    static etcd::Client *WithSSL(std::string const & etcd_url,
+           std::string const & ca,
+           std::string const & cert = "",
+           std::string const & key = "",
+           std::string const & load_balancer = "round_robin");
+
     /**
      * Sends a get request to the etcd server
      * @param key is the key to be read

security-config/.gitignore

@@ -0,0 +1,4 @@
+!.gitignore
+!openssl.cnf
+!setup-ca.sh
+!reset-ca.sh

security-config/openssl.cnf

@@ -0,0 +1,75 @@
+# etcd OpenSSL configuration file.
+#
+# Referred from https://github.com/kelseyhightower/etcd-production-setup/blob/master/openssl.cnf
+#
+SAN = "IP:127.0.0.1"
+dir = .
+
+[ ca ]
+default_ca = etcd_ca
+
+[ etcd_ca ]
+certs            = $dir/certs
+certificate      = $dir/certs/etcd-ca.crt
+crl              = $dir/crl.pem
+crl_dir          = $dir/crl
+crlnumber        = $dir/crlnumber
+database         = $dir/index.txt
+email_in_dn      = no
+new_certs_dir    = $dir/newcerts
+private_key      = $dir/private/etcd-ca.key
+serial           = $dir/serial
+RANDFILE         = $dir/private/.rand
+name_opt         = ca_default
+cert_opt         = ca_default
+default_days     = 3650
+default_crl_days = 30
+default_md       = sha512
+preserve         = no
+policy           = policy_etcd
+
+[ policy_etcd ]
+organizationName = optional
+commonName       = supplied
+
+[ req ]
+default_bits       = 4096
+default_keyfile    = privkey.pem
+distinguished_name = req_distinguished_name
+attributes         = req_attributes
+x509_extensions    = v3_ca
+string_mask        = utf8only
+req_extensions     = etcd_client
+
+[ req_distinguished_name ]
+countryName                = Country Name (2 letter code)
+countryName_default        = US
+countryName_min            = 2
+countryName_max            = 2
+commonName                 = Common Name (FQDN) 
+0.organizationName         = Organization Name (eg, company)
+0.organizationName_default = etcd-ca
+
+[ req_attributes ]
+
+[ v3_ca ]
+basicConstraints       = CA:true
+keyUsage               = keyCertSign,cRLSign
+subjectKeyIdentifier   = hash
+
+[ etcd_client ]
+basicConstraints       = CA:FALSE
+extendedKeyUsage       = clientAuth
+keyUsage               = digitalSignature, keyEncipherment
+
+[ etcd_peer ]
+basicConstraints       = CA:FALSE
+extendedKeyUsage       = clientAuth, serverAuth
+keyUsage               = digitalSignature, keyEncipherment
+subjectAltName         = ${ENV::SAN}
+
+[ etcd_server ]
+basicConstraints       = CA:FALSE
+extendedKeyUsage       = clientAuth, serverAuth
+keyUsage               = digitalSignature, keyEncipherment
+subjectAltName         = ${ENV::SAN} 

security-config/reset-ca.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# generate ca certificate for etcd
+#
+# referred from: https://github.com/kelseyhightower/etcd-production-setup
+
+set -x
+set -e
+set -o pipefail
+
+shopt -s extglob
+
+ROOT=$(dirname "${BASH_SOURCE[0]}")
+
+pushd $ROOT
+
+rm -rf !(setup-ca.sh|reset-ca.sh|openssl.cnf)
+
+popd # $ROOT
+
+set +x
+set +e
+set +o pipefail

security-config/setup-ca.sh

@@ -0,0 +1,105 @@
+#!/bin/bash
+#
+# generate ca certificate for etcd
+#
+# referred from: https://github.com/kelseyhightower/etcd-production-setup
+
+set -x
+set -e
+set -o pipefail
+
+ROOT=$(dirname "${BASH_SOURCE[0]}")
+
+pushd $ROOT
+
+touch index.txt
+echo '01' > serial
+
+mkdir -p private
+mkdir -p certs
+mkdir -p newcerts
+
+# Create the CA Certificate and Key
+openssl req -config ./openssl.cnf -new -x509 -extensions v3_ca \
+  -keyout private/ca.key -out certs/ca.crt \
+  -passin pass:etcd-ca -passout pass:etcd-ca \
+  -subj "/C=US/ST=CA/L=CA/O=etcd-ca/CN=ca.etcd.example.com/emailAddress=ca.etcd.example.com"
+
+# Verify the CA Certificate
+openssl x509 -in certs/ca.crt -noout -text
+
+# Create an etcd server certificate
+# If you want cert verification to work with IPs in addition to hostnames, be sure to set the SAN env var:
+# export SAN="IP:127.0.0.1, IP:10.0.1.10"
+export SAN="IP:127.0.0.1"
+
+openssl req -config openssl.cnf -new -nodes \
+  -keyout private/etcd0.example.com.key -out etcd0.example.com.csr \
+  -subj "/C=US/ST=CA/L=CA/O=etcd-ca/CN=etcd0.example.com/emailAddress=ca.etcd.example.com"
+
+# Sign the cert
+openssl ca -batch -config openssl.cnf -extensions etcd_server \
+  -passin pass:etcd-ca \
+  -keyfile private/ca.key \
+  -cert certs/ca.crt \
+  -out certs/etcd0.example.com.crt -infiles etcd0.example.com.csr
+
+# Verify the etcd Server Certificate
+openssl x509 -in certs/etcd0.example.com.crt -noout -text
+
+# Create an etcd client certificate
+unset SAN
+
+openssl req -config openssl.cnf -new -nodes \
+  -keyout private/etcd-client.key -out etcd-client.csr \
+  -subj "/C=US/ST=CA/L=CA/O=etcd-ca/CN=etcd_client/emailAddress=ca.etcd.example.com"
+
+openssl ca -batch -config openssl.cnf -extensions etcd_client \
+  -passin pass:etcd-ca \
+  -keyfile private/ca.key \
+  -cert certs/ca.crt \
+  -out certs/etcd-client.crt -infiles etcd-client.csr
+
+
+# Configuring etcd for SSL
+
+# Configure etcd
+
+# $ etcd --advertise-client-urls https://etcd0.example.com:2379 \
+#   --listen-client-urls https://10.0.1.10:2379 \
+#   --cert-file etcd0.example.com.crt \
+#   --key-file etcd0.example.com.key
+
+# Configuring etcd clients for SSL
+
+# cURL
+# $ curl --cacert ca.crt -XPUT -v https://etcd0.example.com:2379/v2/keys/foo -d value=bar
+# $ curl --cacert ca.crt -v https://etcd0.example.com:2379/v2/keys
+
+# etcdctl
+# $ etcdctl -C https://etcd0.example.com:2379 --ca-file ca.crt set foo bar 
+# $ etcdctl -C https://etcd0.example.com:2379 --ca-file ca.crt get foo
+
+# Configuring etcd for client auth
+# $ etcd --advertise-client-urls https://etcd0.example.com:2379 \
+#   --listen-client-urls https://10.0.1.10:2379 \
+#   --cert-file etcd0.example.com.crt \
+#   --key-file etcd0.example.com.key \
+#   --client-cert-auth --trusted-ca-file ca.crt \
+#
+# Notice the usage of the `--client-cert-auth` and `--trusted-ca-file` flag. This is what enables client auth.
+
+# Configuring etcd clients for client auth
+
+# etcdctl
+# $ etcdctl -C https://etcd0.example.com:2379 \
+#   --cert etcd-client.crt \
+#   --key etcd-client.key \
+#   --cacert ca.crt \
+#   get foo
+
+popd # $ROOT
+
+set +x
+set +e
+set +o pipefail

src/Client.cpp

@@ -11,12 +11,14 @@
 #endif
 
 #include <iostream>
+#include <fstream>
 #include <limits>
 #include <memory>
 
 #include <boost/algorithm/string.hpp>
 
 #include <grpc++/grpc++.h>
+#include <grpc++/security/credentials.h>
 #include "proto/rpc.grpc.pb.h"
 #include "proto/v3lock.grpc.pb.h"
 
@@ -130,6 +132,27 @@ const bool authenticate(std::shared_ptr<grpc::Channel> const &channel,
   }
 }
 
+static std::string read_from_file(std::string const &filename) {
+  std::ifstream file(filename.c_str(), std::ios::in);
+  if (file.is_open()) {
+    std::stringstream ss;
+		ss << file.rdbuf ();
+		file.close ();
+		return ss.str ();
+  }
+  return std::string{};
+}
+
+static grpc::SslCredentialsOptions make_ssl_credentials(std::string const &ca,
+                                                      std::string const &cert,
+                                                      std::string const &key) {
+  grpc::SslCredentialsOptions options;
+  options.pem_root_certs = read_from_file(ca);
+  options.pem_cert_chain = read_from_file(cert);
+  options.pem_private_key = read_from_file(key);
+  return options;
+}
+
 }
 }
 
@@ -195,6 +218,47 @@ etcd::Client::Client(std::string const & address,
   stubs->lockServiceStub = Lock::NewStub(this->channel);
 }
 
+etcd::Client *etcd::Client::WithUser(std::string const & etcd_url,
+           std::string const & username,
+           std::string const & password,
+           std::string const & load_balancer) {
+  return new etcd::Client(etcd_url, username, password, load_balancer);
+}
+
+etcd::Client::Client(std::string const & address,
+                     std::string const & ca,
+                     std::string const & cert,
+                     std::string const & key,
+                     std::string const & load_balancer)
+{
+  // create channels
+  std::string const addresses = etcd::detail::strip_and_resolve_addresses(address);
+  grpc::ChannelArguments grpc_args;
+  grpc_args.SetMaxSendMessageSize(std::numeric_limits<int>::max());
+  grpc_args.SetMaxReceiveMessageSize(std::numeric_limits<int>::max());
+  std::shared_ptr<grpc::ChannelCredentials> creds = grpc::SslCredentials(
+      etcd::detail::make_ssl_credentials(ca, cert, key));
+  grpc_args.SetLoadBalancingPolicyName(load_balancer);
+  this->channel = grpc::CreateCustomChannel(addresses, creds, grpc_args);
+
+  std::cout << "this->channel : " << this->channel;
+
+  // setup stubs
+  stubs.reset(new EtcdServerStubs{});
+  stubs->kvServiceStub = KV::NewStub(this->channel);
+  stubs->watchServiceStub= Watch::NewStub(this->channel);
+  stubs->leaseServiceStub= Lease::NewStub(this->channel);
+  stubs->lockServiceStub = Lock::NewStub(this->channel);
+}
+
+etcd::Client *etcd::Client::WithSSL(std::string const & etcd_url,
+           std::string const & ca,
+           std::string const & cert,
+           std::string const & key,
+           std::string const & load_balancer) {
+  return new etcd::Client(etcd_url, ca, cert, key, load_balancer);
+}
+
 pplx::task<etcd::Response> etcd::Client::get(std::string const & key)
 {
   etcdv3::ActionParameters params;

tst/AuthTest.cpp

@@ -8,15 +8,15 @@
 
 TEST_CASE("setup with auth")
 {
-  etcd::Client etcd("http://127.0.0.1:2379", "root", "root");
+  etcd::Client *etcd = etcd::Client::WithUser("http://127.0.0.1:2379", "root", "root");
-  etcd.rmdir("/test", true).wait();
+  etcd->rmdir("/test", true).wait();
 }
 
 TEST_CASE("add a new key after authenticate")
 {
-  etcd::Client etcd("http://127.0.0.1:2379", "root", "root");
+  etcd::Client *etcd = etcd::Client::WithUser("http://127.0.0.1:2379", "root", "root");
-  etcd.rmdir("/test", true).wait();
+  etcd->rmdir("/test", true).wait();
-  etcd::Response resp = etcd.add("/test/key1", "42").get();
+  etcd::Response resp = etcd->add("/test/key1", "42").get();
   REQUIRE(0 == resp.error_code());
   CHECK("create" == resp.action());
   etcd::Value const & val = resp.value();
@@ -26,24 +26,24 @@ TEST_CASE("add a new key after authenticate")
   CHECK(0 < val.created_index());
   CHECK(0 < val.modified_index());
   CHECK(0 < resp.index());
-  CHECK(105 == etcd.add("/test/key1", "43").get().error_code()); // Key already exists
+  CHECK(105 == etcd->add("/test/key1", "43").get().error_code()); // Key already exists
-  CHECK(105 == etcd.add("/test/key1", "42").get().error_code()); // Key already exists
+  CHECK(105 == etcd->add("/test/key1", "42").get().error_code()); // Key already exists
-  CHECK("Key already exists" == etcd.add("/test/key1", "42").get().error_message());
+  CHECK("Key already exists" == etcd->add("/test/key1", "42").get().error_message());
 }
 
 TEST_CASE("read a value from etcd")
 {
-  etcd::Client etcd("http://127.0.0.1:2379", "root", "root");
+  etcd::Client *etcd = etcd::Client::WithUser("http://127.0.0.1:2379", "root", "root");
-  etcd::Response resp = etcd.get("/test/key1").get();
+  etcd::Response resp = etcd->get("/test/key1").get();
   CHECK("get" == resp.action());
   REQUIRE(resp.is_ok());
   REQUIRE(0 == resp.error_code());
   CHECK("42" == resp.value().as_string());
-  CHECK("" == etcd.get("/test").get().value().as_string()); // key points to a directory
+  CHECK("" == etcd->get("/test").get().value().as_string()); // key points to a directory
 }
 
 TEST_CASE("cleanup")
 {
-  etcd::Client etcd("http://127.0.0.1:2379", "root", "root");
+  etcd::Client *etcd = etcd::Client::WithUser("http://127.0.0.1:2379", "root", "root");
-  REQUIRE(0 == etcd.rmdir("/test", true).get().error_code());
+  REQUIRE(0 == etcd->rmdir("/test", true).get().error_code());
 }

tst/EtcdSyncTest.cpp

@@ -12,7 +12,6 @@ TEST_CASE("sync operations")
   etcd::SyncClient etcd(etcd_uri);
   etcd.rmdir("/test", true);
 
-
   // add
   CHECK(0 == etcd.add("/test/key1", "42").error_code());
   CHECK(105 == etcd.add("/test/key1", "42").error_code()); // Key already exists

tst/SecurityChannelTest.cpp

@@ -0,0 +1,52 @@
+#define CATCH_CONFIG_MAIN
+#include <catch.hpp>
+
+#include <iostream>
+
+#include "etcd/Client.hpp"
+
+static std::string ca = "security-config/certs/ca.crt";
+static std::string cert = "security-config/certs/etcd0.example.com.crt";
+static std::string key = "security-config/private/etcd0.example.com.key";
+
+TEST_CASE("setup with auth")
+{
+  etcd::Client *etcd = etcd::Client::WithSSL("https://127.0.0.1:2379", ca, cert, key);
+  etcd->rmdir("/test", true).wait();
+}
+
+TEST_CASE("add a new key after authenticate")
+{
+  etcd::Client *etcd = etcd::Client::WithSSL("https://127.0.0.1:2379", ca, cert, key);
+  etcd->rmdir("/test", true).wait();
+  etcd::Response resp = etcd->add("/test/key1", "42").get();
+  REQUIRE(0 == resp.error_code());
+  CHECK("create" == resp.action());
+  etcd::Value const & val = resp.value();
+  CHECK("42" == val.as_string());
+  CHECK("/test/key1" == val.key());
+  CHECK(!val.is_dir());
+  CHECK(0 < val.created_index());
+  CHECK(0 < val.modified_index());
+  CHECK(0 < resp.index());
+  CHECK(105 == etcd->add("/test/key1", "43").get().error_code()); // Key already exists
+  CHECK(105 == etcd->add("/test/key1", "42").get().error_code()); // Key already exists
+  CHECK("Key already exists" == etcd->add("/test/key1", "42").get().error_message());
+}
+
+TEST_CASE("read a value from etcd")
+{
+  etcd::Client *etcd = etcd::Client::WithSSL("https://127.0.0.1:2379", ca, cert, key);
+  etcd::Response resp = etcd->get("/test/key1").get();
+  CHECK("get" == resp.action());
+  REQUIRE(resp.is_ok());
+  REQUIRE(0 == resp.error_code());
+  CHECK("42" == resp.value().as_string());
+  CHECK("" == etcd->get("/test").get().value().as_string()); // key points to a directory
+}
+
+TEST_CASE("cleanup")
+{
+  etcd::Client *etcd = etcd::Client::WithSSL("https://127.0.0.1:2379", ca, cert, key);
+  REQUIRE(0 == etcd->rmdir("/test", true).get().error_code());
+}