Merge pull request #274 from ZhangYaoFu/consul_naming_service

add consul naming service

src/brpc/global.cpp

@@ -28,6 +28,7 @@
 #include "brpc/policy/list_naming_service.h"
 #include "brpc/policy/domain_naming_service.h"
 #include "brpc/policy/remote_file_naming_service.h"
+#include "brpc/policy/consul_naming_service.h"
 
 // Load Balancers
 #include "brpc/policy/round_robin_load_balancer.h"
@@ -105,6 +106,7 @@ struct GlobalExtensions {
     ListNamingService lns;
     DomainNamingService dns;
     RemoteFileNamingService rfns;
+    ConsulNamingService cns;
 
     RoundRobinLoadBalancer rr_lb;
     WeightedRoundRobinLoadBalancer wrr_lb;
@@ -317,6 +319,7 @@ static void GlobalInitializeOrDieImpl() {
     NamingServiceExtension()->RegisterOrDie("list", &g_ext->lns);
     NamingServiceExtension()->RegisterOrDie("http", &g_ext->dns);
     NamingServiceExtension()->RegisterOrDie("remotefile", &g_ext->rfns);
+    NamingServiceExtension()->RegisterOrDie("consul", &g_ext->cns);
 
     // Load Balancers
     LoadBalancerExtension()->RegisterOrDie("rr", &g_ext->rr_lb);

src/brpc/policy/consul_naming_service.cpp

+// Copyright (c) 2014 Baidu, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Authors: Yaofu Zhang (zhangyaofu@qiyi.com)
+
+#include <gflags/gflags.h>
+#include <string>                                       // std::string
+#include <set>                                          // std::set
+#include "butil/string_printf.h"
+#include "butil/third_party/rapidjson/document.h"
+#include "butil/third_party/rapidjson/stringbuffer.h"
+#include "butil/third_party/rapidjson/prettywriter.h"
+#include "butil/time/time.h"
+#include "bthread/bthread.h"
+#include "brpc/log.h"
+#include "brpc/channel.h"
+#include "brpc/policy/file_naming_service.h"
+#include "brpc/policy/consul_naming_service.h"
+
+
+namespace brpc {
+namespace policy {
+
+DEFINE_string(consul_agent_addr, "http://127.0.0.1:8500",
+              "The query string of request consul for discovering service.");
+DEFINE_string(consul_service_discovery_url,
+              "/v1/health/service/",
+              "The url of consul for discovering service.");
+DEFINE_string(consul_url_parameter, "?stale&passing",
+              "The query string of request consul for discovering service.");
+DEFINE_int32(consul_connect_timeout_ms, 200,
+             "Timeout for creating connections to consul in milliseconds");
+DEFINE_int32(consul_blocking_query_wait_secs, 60,
+             "Maximum duration for the blocking request in secs.");
+DEFINE_bool(consul_enable_degrade_to_file_naming_service, false,
+            "Use local backup file when consul cannot connect");
+DEFINE_string(consul_file_naming_service_dir, "",
+    "When it degraded to file naming service, the file with name of the "
+    "service name will be searched in this dir to use.");
+DEFINE_int32(consul_retry_interval_ms, 500,
+             "Wait so many milliseconds before retry when error happens");
+
+constexpr char kConsulIndex[] = "X-Consul-Index";
+
+std::string RapidjsonValueToString(const rapidjson::Value& value) {
+    rapidjson::StringBuffer buffer;
+    rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(buffer);
+    value.Accept(writer);
+    return buffer.GetString();
+}
+
+int ConsulNamingService::DegradeToOtherServiceIfNeeded(const char* service_name,
+                                                       std::vector<ServerNode>* servers) {
+    if (FLAGS_consul_enable_degrade_to_file_naming_service && !_backup_file_loaded) {
+        _backup_file_loaded = true;
+        const std::string file(FLAGS_consul_file_naming_service_dir + service_name);
+        LOG(INFO) << "Load server list from " << file;
+        FileNamingService fns;
+        return fns.GetServers(file.c_str(), servers);
+    }
+    return -1;
+}
+
+int ConsulNamingService::GetServers(const char* service_name,
+                                    std::vector<ServerNode>* servers) {
+    if (!_consul_connected) {
+        ChannelOptions opt;
+        opt.protocol = PROTOCOL_HTTP;
+        opt.connect_timeout_ms = FLAGS_consul_connect_timeout_ms;
+        opt.timeout_ms = (FLAGS_consul_blocking_query_wait_secs + 10) * butil::Time::kMillisecondsPerSecond;
+        if (_channel.Init(FLAGS_consul_agent_addr.c_str(), "rr", &opt) != 0) {
+            LOG(ERROR) << "Fail to init channel to consul at " << FLAGS_consul_agent_addr;
+            return DegradeToOtherServiceIfNeeded(service_name, servers);
+        }
+        _consul_connected = true;
+    }
+
+    if (_consul_url.empty()) {
+        _consul_url.append(FLAGS_consul_service_discovery_url);
+        _consul_url.append(service_name);
+        _consul_url.append(FLAGS_consul_url_parameter);
+    }
+
+    servers->clear();
+    std::string consul_url(_consul_url);
+    if (!_consul_index.empty()) {
+        butil::string_appendf(&consul_url, "&index=%s&wait=%ds", _consul_index.c_str(),
+                              FLAGS_consul_blocking_query_wait_secs);
+    }
+
+    Controller cntl;
+    cntl.http_request().uri() = consul_url;
+    _channel.CallMethod(NULL, &cntl, NULL, NULL, NULL);
+    if (cntl.Failed()) {
+        LOG(ERROR) << "Fail to access " << consul_url << ": "
+                   << cntl.ErrorText();
+        return DegradeToOtherServiceIfNeeded(service_name, servers);
+    }
+
+    const std::string* index = cntl.http_response().GetHeader(kConsulIndex);
+    if (index != nullptr) {
+        if (*index == _consul_index) {
+            LOG_EVERY_N(INFO, 100) << "There is no service changed for the list of "
+                                    << service_name
+                                    << ", consul_index: " << _consul_index;
+            return -1;
+        }
+    } else {
+        LOG(ERROR) << "Failed to parse consul index of " << service_name << ".";
+        return -1;
+    }
+
+    // Sort/unique the inserted vector is faster, but may have a different order
+    // of addresses from the file. To make assertions in tests easier, we use
+    // set to de-duplicate and keep the order.
+    std::set<ServerNode> presence;
+
+    rapidjson::Document services;
+    services.Parse(cntl.response_attachment().to_string().c_str());
+    if (!services.IsArray()) {
+        LOG(ERROR) << "The consul's response for "
+                   << service_name << " is not a json array";
+        return -1;
+    }
+
+    for (rapidjson::SizeType i = 0; i < services.Size(); ++i) {
+        if (!services[i].HasMember("Service")) {
+            LOG(ERROR) << "No service info in node: "
+                       << RapidjsonValueToString(services[i]);
+            continue;
+        }
+
+        const rapidjson::Value& service = services[i]["Service"];
+        if (!service.HasMember("Address") ||
+            !service["Address"].IsString() ||
+            !service.HasMember("Port") ||
+            !service["Port"].IsUint()) {
+            LOG(ERROR) << "Service with no valid address or port: "
+                       << RapidjsonValueToString(service);
+            continue;
+        }
+
+        butil::EndPoint end_point;
+        if (str2endpoint(service["Address"].GetString(),
+                         service["Port"].GetUint(),
+                         &end_point) != 0) {
+            LOG(ERROR) << "Service with illegal address or port: "
+                       << RapidjsonValueToString(service);
+            continue;
+        }
+
+        ServerNode node;
+        node.addr = end_point;
+        if (service.HasMember("Tags")) {
+            if (service["Tags"].IsArray()) {
+                if (service["Tags"].Size() > 0) {
+                    // Tags in consul is an array, here we only use the first one.
+                    if (service["Tags"][0].IsString()) {
+                        node.tag = service["Tags"][0].GetString();
+                    } else {
+                        LOG(ERROR) << "First tag returned by consul is not string, service: "
+                                   << RapidjsonValueToString(service);
+                        continue;
+                    }
+                }
+            } else {
+                LOG(ERROR) << "Service tags returned by consul is not json array, service: "
+                           << RapidjsonValueToString(service);
+                continue;
+            }
+        }
+
+        if (presence.insert(node).second) {
+            servers->push_back(node);
+        } else {
+            RPC_VLOG << "Duplicated server=" << node;
+        }
+    }
+
+    _consul_index = *index;
+
+    if (servers->empty() && !services.Empty()) {
+        LOG(ERROR) << "All service about " << service_name
+                   << " from consul is invalid, refuse to update servers";
+          return -1;
+    }
+
+    RPC_VLOG << "Got " << servers->size()
+             << (servers->size() > 1 ? " servers" : " server")
+             << " from " << service_name;
+    return 0;
+}
+
+int ConsulNamingService::RunNamingService(const char* service_name,
+                                          NamingServiceActions* actions) {
+    std::vector<ServerNode> servers;
+    bool ever_reset = false;
+    for (;;) {
+        servers.clear();
+        const int rc = GetServers(service_name, &servers);
+        if (rc == 0) {
+            ever_reset = true;
+            actions->ResetServers(servers);
+        } else {
+            if (!ever_reset) {
+                // ResetServers must be called at first time even if GetServers
+                // failed, to wake up callers to `WaitForFirstBatchOfServers'
+                ever_reset = true;
+                servers.clear();
+                actions->ResetServers(servers);
+            }
+            if (bthread_usleep(std::max(FLAGS_consul_retry_interval_ms, 1) * butil::Time::kMillisecondsPerSecond) < 0) {
+                if (errno == ESTOP) {
+                    RPC_VLOG << "Quit NamingServiceThread=" << bthread_self();
+                    return 0;
+                }
+                PLOG(FATAL) << "Fail to sleep";
+                return -1;
+            }
+        }
+    }
+    CHECK(false);
+    return -1;
+}
+
+
+void ConsulNamingService::Describe(std::ostream& os,
+                                   const DescribeOptions&) const {
+    os << "consul";
+    return;
+}
+
+NamingService* ConsulNamingService::New() const {
+    return new ConsulNamingService;
+}
+
+void ConsulNamingService::Destroy() {
+    delete this;
+}
+
+}  // namespace policy
+} // namespace brpc

src/brpc/policy/consul_naming_service.h

+// Copyright (c) 2014 Baidu, Inc.G
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Authors: Yaofu Zhang (zhangyaofu@qiyi.com)
+
+#ifndef  BRPC_POLICY_CONSUL_NAMING_SERVICE
+#define  BRPC_POLICY_CONSUL_NAMING_SERVICE
+
+#include "brpc/naming_service.h"
+#include "brpc/channel.h"
+
+
+namespace brpc {
+class Channel;
+namespace policy {
+
+class ConsulNamingService : public NamingService {
+private:
+    int RunNamingService(const char* service_name,
+                         NamingServiceActions* actions);
+
+    int GetServers(const char* service_name,
+                   std::vector<ServerNode>* servers);
+
+    void Describe(std::ostream& os, const DescribeOptions&) const;
+
+    NamingService* New() const;
+
+    int DegradeToOtherServiceIfNeeded(const char* service_name,
+                                      std::vector<ServerNode>* servers);
+
+    void Destroy();
+
+private:
+    Channel _channel;
+    std::string _consul_index;
+    std::string _consul_url;
+    bool _backup_file_loaded = false;
+    bool _consul_connected = false;
+};
+
+}  // namespace policy
+} // namespace brpc
+
+
+#endif  //BRPC_POLICY_CONSUL_NAMING_SERVICE

src/brpc/policy/file_naming_service.h

@@ -24,6 +24,7 @@ namespace brpc {
 namespace policy {
 
 class FileNamingService : public NamingService {
+friend class ConsulNamingService;
 private:
     int RunNamingService(const char* service_name,
                          NamingServiceActions* actions);

test/brpc_naming_service_unittest.cpp

@@ -6,9 +6,11 @@
 #include <vector>
 #include "butil/string_printf.h"
 #include "butil/files/temp_file.h"
+#include "bthread/bthread.h"
 #ifdef BAIDU_INTERNAL
 #include "brpc/policy/baidu_naming_service.h"
 #endif
+#include "brpc/policy/consul_naming_service.h"
 #include "brpc/policy/domain_naming_service.h"
 #include "brpc/policy/file_naming_service.h"
 #include "brpc/policy/list_naming_service.h"
@@ -16,6 +18,17 @@
 #include "echo.pb.h"
 #include "brpc/server.h"
 
+
+namespace brpc {
+namespace policy {
+
+DECLARE_bool(consul_enable_degrade_to_file_naming_service);
+DECLARE_string(consul_file_naming_service_dir);
+DECLARE_string(consul_service_discovery_url);
+
+} // policy
+} // brpc
+
 namespace {
 TEST(NamingServiceTest, sanity) {
     std::vector<brpc::ServerNode> servers;
@@ -202,4 +215,203 @@ TEST(NamingServiceTest, remotefile) {
         ASSERT_EQ(expected_servers[i], servers[i]);
     }
 }
+
+class ConsulNamingServiceImpl : public test::UserNamingService {
+public:
+  ConsulNamingServiceImpl() : list_names_count(0), touch_count(0) {
+  }
+    ~ConsulNamingServiceImpl() { }
+    void ListNames(google::protobuf::RpcController* cntl_base,
+                   const test::HttpRequest*,
+                   test::HttpResponse*,
+                   google::protobuf::Closure* done) {
+        brpc::ClosureGuard done_guard(done);
+        brpc::Controller* cntl = (brpc::Controller*)cntl_base;
+        cntl->http_response().SetHeader("X-Consul-Index", "1");
+        cntl->response_attachment().append(
+            R"([
+                {
+                    "Node": {
+                        "ID": "44454c4c-4e00-1050-8052-b7c04f4b5931",
+                        "Node": "sh-qs-10.121.36.189",
+                        "Address": "10.121.36.189",
+                        "Datacenter": "shjj",
+                        "TaggedAddresses": {
+                            "lan": "10.121.36.189",
+                            "wan": "10.121.36.189"
+                        },
+                        "Meta": {
+                            "consul-network-segment": ""
+                        },
+                        "CreateIndex": 4820296,
+                        "ModifyIndex": 4823818
+                    },
+                    "Service": {
+                        "ID": "10.121.36.189_8003_qs_show_leaf",
+                        "Service": "qs_show_leaf",
+                        "Tags": ["1"],
+                        "Address": "10.121.36.189",
+                        "Port": 8003,
+                        "EnableTagOverride": false,
+                        "CreateIndex": 6515285,
+                        "ModifyIndex": 6515285
+                    },
+                    "Checks": [
+                        {
+                            "Node": "sh-qs-10.121.36.189",
+                            "CheckID": "serfHealth",
+                            "Name": "Serf Health Status",
+                            "Status": "passing",
+                            "Notes": "",
+                            "Output": "Agent alive and reachable",
+                            "ServiceID": "",
+                            "ServiceName": "",
+                            "ServiceTags": [ ],
+                            "CreateIndex": 4820296,
+                            "ModifyIndex": 4820296
+                        },
+                        {
+                            "Node": "sh-qs-10.121.36.189",
+                            "CheckID": "service:10.121.36.189_8003_qs_show_leaf",
+                            "Name": "Service 'qs_show_leaf' check",
+                            "Status": "passing",
+                            "Notes": "",
+                            "Output": "TCP connect 10.121.36.189:8003: Success",
+                            "ServiceID": "10.121.36.189_8003_qs_show_leaf",
+                            "ServiceName": "qs_show_leaf",
+                            "ServiceTags": [ ],
+                            "CreateIndex": 6515285,
+                            "ModifyIndex": 6702198
+                        }
+                    ]
+                },
+                {
+                    "Node": {
+                        "ID": "44454c4c-4b00-1050-8052-b6c04f4b5931",
+                        "Node": "sh-qs-10.121.36.190",
+                        "Address": "10.121.36.190",
+                        "Datacenter": "shjj",
+                        "TaggedAddresses": {
+                            "lan": "10.121.36.190",
+                            "wan": "10.121.36.190"
+                        },
+                        "Meta": {
+                            "consul-network-segment": ""
+                        },
+                        "CreateIndex": 4820296,
+                        "ModifyIndex": 4823751
+                    },
+                    "Service": {
+                        "ID": "10.121.36.190_8003_qs_show_leaf",
+                        "Service": "qs_show_leaf",
+                        "Tags": ["2"],
+                        "Address": "10.121.36.190",
+                        "Port": 8003,
+                        "EnableTagOverride": false,
+                        "CreateIndex": 6515635,
+                        "ModifyIndex": 6515635
+                    },
+                    "Checks": [
+                        {
+                            "Node": "sh-qs-10.121.36.190",
+                            "CheckID": "serfHealth",
+                            "Name": "Serf Health Status",
+                            "Status": "passing",
+                            "Notes": "",
+                            "Output": "Agent alive and reachable",
+                            "ServiceID": "",
+                            "ServiceName": "",
+                            "ServiceTags": [ ],
+                            "CreateIndex": 4820296,
+                            "ModifyIndex": 4820296
+                        },
+                        {
+                            "Node": "sh-qs-10.121.36.190",
+                            "CheckID": "service:10.121.36.190_8003_qs_show_leaf",
+                            "Name": "Service 'qs_show_leaf' check",
+                            "Status": "passing",
+                            "Notes": "",
+                            "Output": "TCP connect 10.121.36.190:8003: Success",
+                            "ServiceID": "10.121.36.190_8003_qs_show_leaf",
+                            "ServiceName": "qs_show_leaf",
+                            "ServiceTags": [ ],
+                            "CreateIndex": 6515635,
+                            "ModifyIndex": 6705515
+                        }
+                    ]
+                }
+            ])");
+        list_names_count.fetch_add(1);
+    }
+    void Touch(google::protobuf::RpcController*,
+               const test::HttpRequest*,
+               test::HttpResponse*,
+               google::protobuf::Closure* done) {
+        brpc::ClosureGuard done_guard(done);
+        touch_count.fetch_add(1);
+    }
+
+    butil::atomic<int64_t> list_names_count;
+    butil::atomic<int64_t> touch_count;
+};
+
+TEST(NamingServiceTest, consul_with_backup_file) {
+    brpc::policy::FLAGS_consul_enable_degrade_to_file_naming_service = true;
+    const char *address_list[] =  {
+        "10.127.0.1:1234",
+        "10.128.0.1:1234",
+        "10.129.0.1:1234",
+    };
+    butil::TempFile tmp_file;
+    const char * service_name = tmp_file.fname();
+    {
+        FILE* fp = fopen(tmp_file.fname(), "w");
+        for (size_t i = 0; i < ARRAY_SIZE(address_list); ++i) {
+            ASSERT_TRUE(fprintf(fp, "%s\n", address_list[i]));
+        }
+        fclose(fp);
+    }
+    std::cout << tmp_file.fname() << std::endl;
+
+    std::vector<brpc::ServerNode> servers;
+    brpc::policy::ConsulNamingService cns;
+    ASSERT_EQ(0, cns.GetServers(service_name, &servers));
+    ASSERT_EQ(ARRAY_SIZE(address_list), servers.size());
+    for (size_t i = 0; i < ARRAY_SIZE(address_list); ++i) {
+        std::ostringstream oss;
+        oss << servers[i];
+        ASSERT_EQ(address_list[i], oss.str()) << "i=" << i;
+    }
+
+    brpc::Server server;
+    ConsulNamingServiceImpl svc;
+    std::string restful_map(brpc::policy::FLAGS_consul_service_discovery_url);
+    restful_map.append("/");
+    restful_map.append(service_name);
+    restful_map.append("   => ListNames");
+    ASSERT_EQ(0, server.AddService(&svc,
+                                   brpc::SERVER_DOESNT_OWN_SERVICE,
+                                   restful_map.c_str()));
+    ASSERT_EQ(0, server.Start("localhost:8500", NULL));
+
+    bthread_usleep(1000000);
+
+    butil::EndPoint n1;
+    ASSERT_EQ(0, butil::str2endpoint("10.121.36.189:8003", &n1));
+    butil::EndPoint n2;
+    ASSERT_EQ(0, butil::str2endpoint("10.121.36.190:8003", &n2));
+    std::vector<brpc::ServerNode> expected_servers;
+    expected_servers.push_back(brpc::ServerNode(n1, "1"));
+    expected_servers.push_back(brpc::ServerNode(n2, "2"));
+    std::sort(expected_servers.begin(), expected_servers.end());
+
+    servers.clear();
+    ASSERT_EQ(0, cns.GetServers(service_name, &servers));
+    ASSERT_EQ(expected_servers.size(), servers.size());
+    std::sort(servers.begin(), servers.end());
+    for (size_t i = 0; i < expected_servers.size(); ++i) {
+        ASSERT_EQ(expected_servers[i], servers[i]);
+    }
+}
+
 } //namespace