diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 6333810..4611b28 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -39,6 +39,7 @@ jobs: libssl-dev \ libz-dev \ lsb-release \ + openssl \ screenfetch \ wget @@ -136,7 +137,7 @@ jobs: -DBUILD_ETCD_TESTS=ON \ -DCMAKE_C_COMPILER_LAUNCHER=ccache \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache - make -j2 + make -j`nproc` sudo make install - name: Setup tmate session @@ -150,16 +151,31 @@ jobs: # use etcd v3 api export ETCDCTL_API="3" - cd build + rm -rf default.etcd /usr/local/bin/etcd & - # tests without auth - ./bin/EtcdSyncTest - ./bin/EtcdTest - ./bin/LockTest - ./bin/WatcherTest + sleep 5 - # tests with auth + # tests without auth + ./build/bin/EtcdSyncTest + ./build/bin/EtcdTest + ./build/bin/LockTest + ./build/bin/WatcherTest + + killall -TERM etcd + sleep 5 + + - name: Authentication Test + run: | + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib + + # use etcd v3 api + export ETCDCTL_API="3" + + rm -rf default.etcd + /usr/local/bin/etcd & + + sleep 5 # for etcd v3.2, v3.3 if [[ "${{ matrix.etcd }}" == v3.2* ]] || [[ "${{ matrix.etcd }}" == v3.3* ]]; @@ -174,7 +190,7 @@ jobs: /usr/local/bin/etcdctl auth enable || true - ./bin/AuthTest + ./build/bin/AuthTest # for etcd v3.2 if [[ "${{ matrix.etcd }}" == v3.2* ]] || [[ "${{ matrix.etcd }}" == v3.3* ]]; @@ -187,6 +203,36 @@ jobs: /usr/local/bin/etcdctl auth disable --user="root" --password="root" || true fi + killall -TERM etcd + sleep 5 + + - name: Transport Security and Authentication Test + run: | + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib + + # use etcd v3 api + export ETCDCTL_API="3" + + # generate CA certificates + ./security-config/reset-ca.sh + ./security-config/setup-ca.sh + + rm -rf default.etcd + /usr/local/bin/etcd \ + --cert-file security-config/certs/etcd0.example.com.crt \ + --key-file security-config/private/etcd0.example.com.key \ + --client-cert-auth \ + --trusted-ca-file security-config/certs/ca.crt \ + --advertise-client-urls=https://127.0.0.1:2379 \ + --listen-client-urls=https://127.0.0.1:2379 & + + sleep 5 + + ./build/bin/SecurityChannelTest + + killall -TERM etcd + sleep 5 + - name: Check ccache run: | ccache --show-stats diff --git a/README.md b/README.md index b8ba98d..26fa766 100644 --- a/README.md +++ b/README.md @@ -160,7 +160,9 @@ be used by default. ### Etcd authentication -Etcd [v3's authentication](https://etcd.io/docs/v3.4.0/learning/design-auth-v3/) is currently +#### v3 authentication + +Etcd [v3 authentication](https://etcd.io/docs/v3.4.0/learning/design-auth-v3/) has been supported. The `Client::Client` could accept a `username` and `password` as arguments and handle the authentication properly. @@ -168,6 +170,13 @@ the authentication properly. etcd::Client etcd("http://127.0.0.1:2379", "root", "root"); ``` +Or the etcd client can be constructed explictly: + +```c++ + etcd::Client *etcd = etcd::Client::WithUser( + "http://127.0.0.1:2379", "root", "root"); +``` + Enabling v3 authentication requires a bit more work for older versions etcd (etcd 3.2.x and etcd 3.3.x). First you need to set the `ETCDCTL_API=3`, then @@ -189,6 +198,43 @@ printf 'root\nroot\n' | /usr/local/bin/etcdctl user add root /usr/local/bin/etcdctl --user="root:root" auth disable ``` +#### transport security + +Etcd [transport security](https://etcd.io/docs/v3.4.0/op-guide/security/) and certificate based +authentication have been supported as well. The `Client::Client` could accept arguments `ca` , +`cert` and `key` for CA cert, cert and private key files for the SSL/TLS transport and authentication. +Note that the later arguments `cert` and `key` could be empty strings or omitted if you just need +secure transport and don't enable certificate-based client authentication (using the `--client-cert-auth` +arguments when launching etcd server). + +```c++ + etcd::Client etcd("https://127.0.0.1:2379", + "example.rootca.cert", "example.cert", "example.key", + "round_robin"); +``` + +Or the etcd client can be constructed explictly: + +```c++ + etcd::Client *etcd = etcd::Client::WithUser( + "https://127.0.0.1:2379", + "example.rootca.cert", "example.cert", "example.key"); +``` + +Using secure transport but not certificated-based client authentication: + +```c++ + etcd::Client *etcd = etcd::Client::WithUser( + "https://127.0.0.1:2379", "example.rootca.cert"); +``` + +For more details about setup about security communication between etcd server and client, please +refer to [transport security](https://etcd.io/docs/v3.4.0/op-guide/security/) in etcd documentation +and [an example](https://github.com/kelseyhightower/etcd-production-setup) about setup etcd with +transport security using openssl. + +We also provide a tool [`setup-ca.sh`](./security-config/setup-ca.sh) as a helper for development and testing. + ### Reading a value You can read a value with the ```get``` method of the clinent instance. The only parameter is the diff --git a/etcd/Client.hpp b/etcd/Client.hpp index 0003eca..5bf73be 100644 --- a/etcd/Client.hpp +++ b/etcd/Client.hpp @@ -50,6 +50,52 @@ namespace etcd std::string const & password, std::string const & load_balancer = "round_robin"); + /** + * Constructs an etcd client object. + * + * @param etcd_url is the url of the etcd server to connect to, like "http://127.0.0.1:2379", + * or multiple url, seperated by ',' or ';'. + * @param username username of etcd auth + * @param password password of etcd auth + * @param load_balancer is the load balance strategy, can be one of round_robin/pick_first/grpclb/xds. + */ + static etcd::Client *WithUser(std::string const & etcd_url, + std::string const & username, + std::string const & password, + std::string const & load_balancer = "round_robin"); + + /** + * Constructs an etcd client object. + * + * @param etcd_url is the url of the etcd server to connect to, like "http://127.0.0.1:2379", + * or multiple url, seperated by ',' or ';'. + * @param ca root CA file for SSL/TLS connection. + * @param cert cert chain file for SSL/TLS authentication, could be empty string. + * @param key private key file for SSL/TLS authentication, could be empty string. + * @param load_balancer is the load balance strategy, can be one of round_robin/pick_first/grpclb/xds. + */ + Client(std::string const & etcd_url, + std::string const & ca, + std::string const & cert, + std::string const & key, + std::string const & load_balancer); + + /** + * Constructs an etcd client object. + * + * @param etcd_url is the url of the etcd server to connect to, like "http://127.0.0.1:2379", + * or multiple url, seperated by ',' or ';'. + * @param ca root CA file for SSL/TLS connection. + * @param cert cert chain file for SSL/TLS authentication, could be empty string. + * @param key private key file for SSL/TLS authentication, could be empty string. + * @param load_balancer is the load balance strategy, can be one of round_robin/pick_first/grpclb/xds. + */ + static etcd::Client *WithSSL(std::string const & etcd_url, + std::string const & ca, + std::string const & cert = "", + std::string const & key = "", + std::string const & load_balancer = "round_robin"); + /** * Sends a get request to the etcd server * @param key is the key to be read diff --git a/security-config/.gitignore b/security-config/.gitignore new file mode 100644 index 0000000..2f1e6fe --- /dev/null +++ b/security-config/.gitignore @@ -0,0 +1,4 @@ +!.gitignore +!openssl.cnf +!setup-ca.sh +!reset-ca.sh diff --git a/security-config/openssl.cnf b/security-config/openssl.cnf new file mode 100644 index 0000000..53e9d62 --- /dev/null +++ b/security-config/openssl.cnf @@ -0,0 +1,75 @@ +# etcd OpenSSL configuration file. +# +# Referred from https://github.com/kelseyhightower/etcd-production-setup/blob/master/openssl.cnf +# +SAN = "IP:127.0.0.1" +dir = . + +[ ca ] +default_ca = etcd_ca + +[ etcd_ca ] +certs = $dir/certs +certificate = $dir/certs/etcd-ca.crt +crl = $dir/crl.pem +crl_dir = $dir/crl +crlnumber = $dir/crlnumber +database = $dir/index.txt +email_in_dn = no +new_certs_dir = $dir/newcerts +private_key = $dir/private/etcd-ca.key +serial = $dir/serial +RANDFILE = $dir/private/.rand +name_opt = ca_default +cert_opt = ca_default +default_days = 3650 +default_crl_days = 30 +default_md = sha512 +preserve = no +policy = policy_etcd + +[ policy_etcd ] +organizationName = optional +commonName = supplied + +[ req ] +default_bits = 4096 +default_keyfile = privkey.pem +distinguished_name = req_distinguished_name +attributes = req_attributes +x509_extensions = v3_ca +string_mask = utf8only +req_extensions = etcd_client + +[ req_distinguished_name ] +countryName = Country Name (2 letter code) +countryName_default = US +countryName_min = 2 +countryName_max = 2 +commonName = Common Name (FQDN) +0.organizationName = Organization Name (eg, company) +0.organizationName_default = etcd-ca + +[ req_attributes ] + +[ v3_ca ] +basicConstraints = CA:true +keyUsage = keyCertSign,cRLSign +subjectKeyIdentifier = hash + +[ etcd_client ] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth +keyUsage = digitalSignature, keyEncipherment + +[ etcd_peer ] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth, serverAuth +keyUsage = digitalSignature, keyEncipherment +subjectAltName = ${ENV::SAN} + +[ etcd_server ] +basicConstraints = CA:FALSE +extendedKeyUsage = clientAuth, serverAuth +keyUsage = digitalSignature, keyEncipherment +subjectAltName = ${ENV::SAN} diff --git a/security-config/reset-ca.sh b/security-config/reset-ca.sh new file mode 100755 index 0000000..aa68044 --- /dev/null +++ b/security-config/reset-ca.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# +# generate ca certificate for etcd +# +# referred from: https://github.com/kelseyhightower/etcd-production-setup + +set -x +set -e +set -o pipefail + +shopt -s extglob + +ROOT=$(dirname "${BASH_SOURCE[0]}") + +pushd $ROOT + +rm -rf !(setup-ca.sh|reset-ca.sh|openssl.cnf) + +popd # $ROOT + +set +x +set +e +set +o pipefail diff --git a/security-config/setup-ca.sh b/security-config/setup-ca.sh new file mode 100755 index 0000000..a51d08d --- /dev/null +++ b/security-config/setup-ca.sh @@ -0,0 +1,105 @@ +#!/bin/bash +# +# generate ca certificate for etcd +# +# referred from: https://github.com/kelseyhightower/etcd-production-setup + +set -x +set -e +set -o pipefail + +ROOT=$(dirname "${BASH_SOURCE[0]}") + +pushd $ROOT + +touch index.txt +echo '01' > serial + +mkdir -p private +mkdir -p certs +mkdir -p newcerts + +# Create the CA Certificate and Key +openssl req -config ./openssl.cnf -new -x509 -extensions v3_ca \ + -keyout private/ca.key -out certs/ca.crt \ + -passin pass:etcd-ca -passout pass:etcd-ca \ + -subj "/C=US/ST=CA/L=CA/O=etcd-ca/CN=ca.etcd.example.com/emailAddress=ca.etcd.example.com" + +# Verify the CA Certificate +openssl x509 -in certs/ca.crt -noout -text + +# Create an etcd server certificate +# If you want cert verification to work with IPs in addition to hostnames, be sure to set the SAN env var: +# export SAN="IP:127.0.0.1, IP:10.0.1.10" +export SAN="IP:127.0.0.1" + +openssl req -config openssl.cnf -new -nodes \ + -keyout private/etcd0.example.com.key -out etcd0.example.com.csr \ + -subj "/C=US/ST=CA/L=CA/O=etcd-ca/CN=etcd0.example.com/emailAddress=ca.etcd.example.com" + +# Sign the cert +openssl ca -batch -config openssl.cnf -extensions etcd_server \ + -passin pass:etcd-ca \ + -keyfile private/ca.key \ + -cert certs/ca.crt \ + -out certs/etcd0.example.com.crt -infiles etcd0.example.com.csr + +# Verify the etcd Server Certificate +openssl x509 -in certs/etcd0.example.com.crt -noout -text + +# Create an etcd client certificate +unset SAN + +openssl req -config openssl.cnf -new -nodes \ + -keyout private/etcd-client.key -out etcd-client.csr \ + -subj "/C=US/ST=CA/L=CA/O=etcd-ca/CN=etcd_client/emailAddress=ca.etcd.example.com" + +openssl ca -batch -config openssl.cnf -extensions etcd_client \ + -passin pass:etcd-ca \ + -keyfile private/ca.key \ + -cert certs/ca.crt \ + -out certs/etcd-client.crt -infiles etcd-client.csr + + +# Configuring etcd for SSL + +# Configure etcd + +# $ etcd --advertise-client-urls https://etcd0.example.com:2379 \ +# --listen-client-urls https://10.0.1.10:2379 \ +# --cert-file etcd0.example.com.crt \ +# --key-file etcd0.example.com.key + +# Configuring etcd clients for SSL + +# cURL +# $ curl --cacert ca.crt -XPUT -v https://etcd0.example.com:2379/v2/keys/foo -d value=bar +# $ curl --cacert ca.crt -v https://etcd0.example.com:2379/v2/keys + +# etcdctl +# $ etcdctl -C https://etcd0.example.com:2379 --ca-file ca.crt set foo bar +# $ etcdctl -C https://etcd0.example.com:2379 --ca-file ca.crt get foo + +# Configuring etcd for client auth +# $ etcd --advertise-client-urls https://etcd0.example.com:2379 \ +# --listen-client-urls https://10.0.1.10:2379 \ +# --cert-file etcd0.example.com.crt \ +# --key-file etcd0.example.com.key \ +# --client-cert-auth --trusted-ca-file ca.crt \ +# +# Notice the usage of the `--client-cert-auth` and `--trusted-ca-file` flag. This is what enables client auth. + +# Configuring etcd clients for client auth + +# etcdctl +# $ etcdctl -C https://etcd0.example.com:2379 \ +# --cert etcd-client.crt \ +# --key etcd-client.key \ +# --cacert ca.crt \ +# get foo + +popd # $ROOT + +set +x +set +e +set +o pipefail diff --git a/src/Client.cpp b/src/Client.cpp index 2c8f10e..70685fd 100644 --- a/src/Client.cpp +++ b/src/Client.cpp @@ -11,12 +11,14 @@ #endif #include +#include #include #include #include #include +#include #include "proto/rpc.grpc.pb.h" #include "proto/v3lock.grpc.pb.h" @@ -130,6 +132,27 @@ const bool authenticate(std::shared_ptr const &channel, } } +static std::string read_from_file(std::string const &filename) { + std::ifstream file(filename.c_str(), std::ios::in); + if (file.is_open()) { + std::stringstream ss; + ss << file.rdbuf (); + file.close (); + return ss.str (); + } + return std::string{}; +} + +static grpc::SslCredentialsOptions make_ssl_credentials(std::string const &ca, + std::string const &cert, + std::string const &key) { + grpc::SslCredentialsOptions options; + options.pem_root_certs = read_from_file(ca); + options.pem_cert_chain = read_from_file(cert); + options.pem_private_key = read_from_file(key); + return options; +} + } } @@ -195,6 +218,47 @@ etcd::Client::Client(std::string const & address, stubs->lockServiceStub = Lock::NewStub(this->channel); } +etcd::Client *etcd::Client::WithUser(std::string const & etcd_url, + std::string const & username, + std::string const & password, + std::string const & load_balancer) { + return new etcd::Client(etcd_url, username, password, load_balancer); +} + +etcd::Client::Client(std::string const & address, + std::string const & ca, + std::string const & cert, + std::string const & key, + std::string const & load_balancer) +{ + // create channels + std::string const addresses = etcd::detail::strip_and_resolve_addresses(address); + grpc::ChannelArguments grpc_args; + grpc_args.SetMaxSendMessageSize(std::numeric_limits::max()); + grpc_args.SetMaxReceiveMessageSize(std::numeric_limits::max()); + std::shared_ptr creds = grpc::SslCredentials( + etcd::detail::make_ssl_credentials(ca, cert, key)); + grpc_args.SetLoadBalancingPolicyName(load_balancer); + this->channel = grpc::CreateCustomChannel(addresses, creds, grpc_args); + + std::cout << "this->channel : " << this->channel; + + // setup stubs + stubs.reset(new EtcdServerStubs{}); + stubs->kvServiceStub = KV::NewStub(this->channel); + stubs->watchServiceStub= Watch::NewStub(this->channel); + stubs->leaseServiceStub= Lease::NewStub(this->channel); + stubs->lockServiceStub = Lock::NewStub(this->channel); +} + +etcd::Client *etcd::Client::WithSSL(std::string const & etcd_url, + std::string const & ca, + std::string const & cert, + std::string const & key, + std::string const & load_balancer) { + return new etcd::Client(etcd_url, ca, cert, key, load_balancer); +} + pplx::task etcd::Client::get(std::string const & key) { etcdv3::ActionParameters params; diff --git a/tst/AuthTest.cpp b/tst/AuthTest.cpp index a6d826a..fa382b5 100644 --- a/tst/AuthTest.cpp +++ b/tst/AuthTest.cpp @@ -8,15 +8,15 @@ TEST_CASE("setup with auth") { - etcd::Client etcd("http://127.0.0.1:2379", "root", "root"); - etcd.rmdir("/test", true).wait(); + etcd::Client *etcd = etcd::Client::WithUser("http://127.0.0.1:2379", "root", "root"); + etcd->rmdir("/test", true).wait(); } TEST_CASE("add a new key after authenticate") { - etcd::Client etcd("http://127.0.0.1:2379", "root", "root"); - etcd.rmdir("/test", true).wait(); - etcd::Response resp = etcd.add("/test/key1", "42").get(); + etcd::Client *etcd = etcd::Client::WithUser("http://127.0.0.1:2379", "root", "root"); + etcd->rmdir("/test", true).wait(); + etcd::Response resp = etcd->add("/test/key1", "42").get(); REQUIRE(0 == resp.error_code()); CHECK("create" == resp.action()); etcd::Value const & val = resp.value(); @@ -26,24 +26,24 @@ TEST_CASE("add a new key after authenticate") CHECK(0 < val.created_index()); CHECK(0 < val.modified_index()); CHECK(0 < resp.index()); - CHECK(105 == etcd.add("/test/key1", "43").get().error_code()); // Key already exists - CHECK(105 == etcd.add("/test/key1", "42").get().error_code()); // Key already exists - CHECK("Key already exists" == etcd.add("/test/key1", "42").get().error_message()); + CHECK(105 == etcd->add("/test/key1", "43").get().error_code()); // Key already exists + CHECK(105 == etcd->add("/test/key1", "42").get().error_code()); // Key already exists + CHECK("Key already exists" == etcd->add("/test/key1", "42").get().error_message()); } TEST_CASE("read a value from etcd") { - etcd::Client etcd("http://127.0.0.1:2379", "root", "root"); - etcd::Response resp = etcd.get("/test/key1").get(); + etcd::Client *etcd = etcd::Client::WithUser("http://127.0.0.1:2379", "root", "root"); + etcd::Response resp = etcd->get("/test/key1").get(); CHECK("get" == resp.action()); REQUIRE(resp.is_ok()); REQUIRE(0 == resp.error_code()); CHECK("42" == resp.value().as_string()); - CHECK("" == etcd.get("/test").get().value().as_string()); // key points to a directory + CHECK("" == etcd->get("/test").get().value().as_string()); // key points to a directory } TEST_CASE("cleanup") { - etcd::Client etcd("http://127.0.0.1:2379", "root", "root"); - REQUIRE(0 == etcd.rmdir("/test", true).get().error_code()); + etcd::Client *etcd = etcd::Client::WithUser("http://127.0.0.1:2379", "root", "root"); + REQUIRE(0 == etcd->rmdir("/test", true).get().error_code()); } diff --git a/tst/EtcdSyncTest.cpp b/tst/EtcdSyncTest.cpp index 549f0ae..21dee7b 100644 --- a/tst/EtcdSyncTest.cpp +++ b/tst/EtcdSyncTest.cpp @@ -12,7 +12,6 @@ TEST_CASE("sync operations") etcd::SyncClient etcd(etcd_uri); etcd.rmdir("/test", true); - // add CHECK(0 == etcd.add("/test/key1", "42").error_code()); CHECK(105 == etcd.add("/test/key1", "42").error_code()); // Key already exists diff --git a/tst/SecurityChannelTest.cpp b/tst/SecurityChannelTest.cpp new file mode 100644 index 0000000..845c55d --- /dev/null +++ b/tst/SecurityChannelTest.cpp @@ -0,0 +1,52 @@ +#define CATCH_CONFIG_MAIN +#include + +#include + +#include "etcd/Client.hpp" + +static std::string ca = "security-config/certs/ca.crt"; +static std::string cert = "security-config/certs/etcd0.example.com.crt"; +static std::string key = "security-config/private/etcd0.example.com.key"; + +TEST_CASE("setup with auth") +{ + etcd::Client *etcd = etcd::Client::WithSSL("https://127.0.0.1:2379", ca, cert, key); + etcd->rmdir("/test", true).wait(); +} + +TEST_CASE("add a new key after authenticate") +{ + etcd::Client *etcd = etcd::Client::WithSSL("https://127.0.0.1:2379", ca, cert, key); + etcd->rmdir("/test", true).wait(); + etcd::Response resp = etcd->add("/test/key1", "42").get(); + REQUIRE(0 == resp.error_code()); + CHECK("create" == resp.action()); + etcd::Value const & val = resp.value(); + CHECK("42" == val.as_string()); + CHECK("/test/key1" == val.key()); + CHECK(!val.is_dir()); + CHECK(0 < val.created_index()); + CHECK(0 < val.modified_index()); + CHECK(0 < resp.index()); + CHECK(105 == etcd->add("/test/key1", "43").get().error_code()); // Key already exists + CHECK(105 == etcd->add("/test/key1", "42").get().error_code()); // Key already exists + CHECK("Key already exists" == etcd->add("/test/key1", "42").get().error_message()); +} + +TEST_CASE("read a value from etcd") +{ + etcd::Client *etcd = etcd::Client::WithSSL("https://127.0.0.1:2379", ca, cert, key); + etcd::Response resp = etcd->get("/test/key1").get(); + CHECK("get" == resp.action()); + REQUIRE(resp.is_ok()); + REQUIRE(0 == resp.error_code()); + CHECK("42" == resp.value().as_string()); + CHECK("" == etcd->get("/test").get().value().as_string()); // key points to a directory +} + +TEST_CASE("cleanup") +{ + etcd::Client *etcd = etcd::Client::WithSSL("https://127.0.0.1:2379", ca, cert, key); + REQUIRE(0 == etcd->rmdir("/test", true).get().error_code()); +}