Merge branch 'master' of ssh://icode.baidu.com:8235/baidu/opensource/baidu-rpc

Makefile

@@ -11,8 +11,9 @@ CXXFLAGS=$(CPPFLAGS) -g -O2 -pipe -Wall -W -fPIC -fstrict-aliasing -Wno-invalid-
 CFLAGS=$(CPPFLAGS) -g -O2 -pipe -Wall -W -fPIC -fstrict-aliasing -Wno-unused-parameter -fno-omit-frame-pointer
 HDRPATHS=-I./src $(addprefix -I, $(HDRS))
 LIBPATHS = $(addprefix -L, $(LIBS))
+COMMA = ,
+SOPATHS = $(addprefix -Wl$(COMMA)-rpath=, $(LIBS))
 SRCEXTS = .c .cc .cpp .proto
-HDREXTS = .h .hpp
 
 #required by butil/crc32.cc to boost performance for 10x
 ifeq ($(shell test $(GCC_VERSION) -ge 40400; echo $$?),0)
@@ -195,7 +196,7 @@ JSON2PB_OBJS = $(addsuffix .o, $(basename $(JSON2PB_SOURCES)))
 BRPC_DIRS = src/brpc src/brpc/details src/brpc/builtin src/brpc/policy
 BRPC_SOURCES = $(foreach d,$(BRPC_DIRS),$(wildcard $(addprefix $(d)/*,$(SRCEXTS))))
 BRPC_PROTOS = $(filter %.proto,$(BRPC_SOURCES))
-BRPC_CFAMILIES = $(filter-out %.proto,$(BRPC_SOURCES))
+BRPC_CFAMILIES = $(filter-out %.proto %.pb.cc,$(BRPC_SOURCES))
 BRPC_OBJS = $(BRPC_PROTOS:.proto=.pb.o) $(addsuffix .o, $(basename $(BRPC_CFAMILIES)))
 
 MCPACK2PB_SOURCES = \
@@ -209,7 +210,7 @@ OBJS=$(BUTIL_OBJS) $(BVAR_OBJS) $(BTHREAD_OBJS) $(JSON2PB_OBJS) $(MCPACK2PB_OBJS
 DEBUG_OBJS = $(OBJS:.o=.dbg.o)
 
 .PHONY:all
-all:  protoc-gen-mcpack libbrpc.a output/include output/lib output/bin
+all:  protoc-gen-mcpack libbrpc.a libbrpc.so output/include output/lib output/bin
 
 .PHONY:debug
 debug: libbrpc.dbg.a
@@ -217,7 +218,7 @@ debug: libbrpc.dbg.a
 .PHONY:clean
 clean:clean_debug
 	@echo "Cleaning"
-	@rm -rf mcpack2pb/generator.o protoc-gen-mcpack libbrpc.a $(OBJS) output/include output/lib output/bin
+	@rm -rf src/mcpack2pb/generator.o protoc-gen-mcpack libbrpc.a libbrpc.so $(OBJS) output/include output/lib output/bin
 
 .PHONY:clean_debug
 clean_debug:
@@ -232,6 +233,10 @@ libbrpc.a:$(BRPC_PROTOS:.proto=.pb.h) $(OBJS)
 	@echo "Packing $@"
 	@ar crs $@ $(OBJS)
 
+libbrpc.so:$(BRPC_PROTOS:.proto=.pb.h) $(OBJS)
+	@echo "Linking $@"
+	$(CXX) -shared -o $@ $(HDRPATHS) $(LIBPATHS) $(SOPATHS) -Xlinker "-(" $(OBJS) -Xlinker "-)" $(STATIC_LINKINGS) $(DYNAMIC_LINKINGS)
+
 libbrpc.dbg.a:$(BRPC_PROTOS:.proto=.pb.h) $(DEBUG_OBJS)
 	@echo "Packing $@"
 	@ar crs $@ $(DEBUG_OBJS)
@@ -244,7 +249,7 @@ output/include:
 	@cp src/idl_options.proto src/idl_options.pb.h $@
 
 .PHONY:output/lib
-output/lib:libbrpc.a
+output/lib:libbrpc.a libbrpc.so
 	@echo "Copying to $@"
 	@mkdir -p $@
 	@cp $^ $@

example/echo_c++/Makefile

@@ -4,8 +4,13 @@ include $(BRPC_PATH)/config.mk
 # 1. Added -fno-omit-frame-pointer: perf/tcmalloc-profiler use frame pointers by default
 # 2. Added -D__const__= : Avoid over-optimizations of TLS variables by GCC>=4.8
 CXXFLAGS = -std=c++0x -g -DNDEBUG -O2 -D__const__= -pipe -W -Wall -Werror -Wno-unused-parameter -fPIC -fno-omit-frame-pointer
-HDRPATHS = -I$(BRPC_PATH)/output/include $(addprefix -I, $(HDRS))
-LIBPATHS = -L$(BRPC_PATH)/output/lib $(addprefix -L, $(LIBS))
+HDRS+=$(BRPC_PATH)/output/include
+LIBS+=$(BRPC_PATH)/output/lib
+HDRPATHS = $(addprefix -I, $(HDRS))
+LIBPATHS = $(addprefix -L, $(LIBS))
+COMMA=,
+SOPATHS=$(addprefix -Wl$(COMMA)-rpath=, $(LIBS))
+
 STATIC_LINKINGS += -lbrpc
 
 CLIENT_SOURCES = client.cpp
@@ -27,11 +32,19 @@ clean:
 
 echo_client:$(PROTO_OBJS) $(CLIENT_OBJS)
 	@echo "Linking $@"
+ifneq ("$(LINK_SO)", "")
+	@$(CXX) $(LIBPATHS) $(SOPATHS) -Xlinker "-(" $^ -Xlinker "-)" $(STATIC_LINKINGS) $(DYNAMIC_LINKINGS) -o $@
+else
 	@$(CXX) $(LIBPATHS) -Xlinker "-(" $^ -Wl,-Bstatic $(STATIC_LINKINGS) -Wl,-Bdynamic -Xlinker "-)" $(DYNAMIC_LINKINGS) -o $@
+endif
 
 echo_server:$(PROTO_OBJS) $(SERVER_OBJS)
 	@echo "Linking $@"
+ifneq ("$(LINK_SO)", "")
+	@$(CXX) $(LIBPATHS) $(SOPATHS) -Xlinker "-(" $^ -Xlinker "-)" $(STATIC_LINKINGS) $(DYNAMIC_LINKINGS) -o $@
+else
 	@$(CXX) $(LIBPATHS) -Xlinker "-(" $^ -Wl,-Bstatic $(STATIC_LINKINGS) -Wl,-Bdynamic -Xlinker "-)" $(DYNAMIC_LINKINGS) -o $@
+endif
 
 %.pb.cc %.pb.h:%.proto
 	@echo "Generating $@"

example/http_c++/Makefile

@@ -8,8 +8,13 @@ CXXFLAGS = -std=c++0x -g -DNDEBUG -O2 -D__const__= -pipe -W -Wall -Werror -Wno-u
 ifeq ($(NEED_GPERFTOOLS), 1)
 	CXXFLAGS+=-DBRPC_ENABLE_CPU_PROFILER -DBRPC_ENABLE_HEAP_PROFILER
 endif
-HDRPATHS = -I$(BRPC_PATH)/output/include $(addprefix -I, $(HDRS))
-LIBPATHS = -L$(BRPC_PATH)/output/lib $(addprefix -L, $(LIBS))
+HDRS+=$(BRPC_PATH)/output/include
+LIBS+=$(BRPC_PATH)/output/lib
+HDRPATHS = $(addprefix -I, $(HDRS))
+LIBPATHS = $(addprefix -L, $(LIBS))
+COMMA=,
+SOPATHS=$(addprefix -Wl$(COMMA)-rpath=, $(LIBS))
+
 STATIC_LINKINGS += -lbrpc
 
 CLIENT_SOURCES = http_client.cpp
@@ -33,15 +38,27 @@ clean:
 
 http_client:$(CLIENT_OBJS)
 	@echo "Linking $@"
+ifneq ("$(LINK_SO)", "")
+	@$(CXX) $(LIBPATHS) $(SOPATHS) -Xlinker "-(" $^ -Xlinker "-)" $(STATIC_LINKINGS) $(DYNAMIC_LINKINGS) -o $@
+else
 	@$(CXX) $(LIBPATHS) -Xlinker "-(" $^ -Wl,-Bstatic $(STATIC_LINKINGS) -Wl,-Bdynamic -Xlinker "-)" $(DYNAMIC_LINKINGS) -o $@
+endif
 
 benchmark_http:$(BENCHMARK_OBJS)
 	@echo "Linking $@"
+ifneq ("$(LINK_SO)", "")
+	@$(CXX) $(LIBPATHS) $(SOPATHS) -Xlinker "-(" $^ -Xlinker "-)" $(STATIC_LINKINGS) $(DYNAMIC_LINKINGS) -o $@
+else
 	@$(CXX) $(LIBPATHS) -Xlinker "-(" $^ -Wl,-Bstatic $(STATIC_LINKINGS) -Wl,-Bdynamic -Xlinker "-)" $(DYNAMIC_LINKINGS) -o $@
+endif
 
 http_server:$(PROTO_OBJS) $(SERVER_OBJS)
 	@echo "Linking $@"
+ifneq ("$(LINK_SO)", "")
+	@$(CXX) $(LIBPATHS) $(SOPATHS) -Xlinker "-(" $^ -Xlinker "-)" $(STATIC_LINKINGS) $(DYNAMIC_LINKINGS) -o $@
+else
 	@$(CXX) $(LIBPATHS) -Xlinker "-(" $^ -Wl,-Bstatic $(STATIC_LINKINGS) -Wl,-Bdynamic -Xlinker "-)" $(DYNAMIC_LINKINGS) -o $@
+endif
 
 %.pb.cc %.pb.h:%.proto
 	@echo "Generating $@"

example/memcache_c++/Makefile

 BRPC_PATH = ../../
 include $(BRPC_PATH)/config.mk
 CXXFLAGS = -std=c++0x -g -DNDEBUG -O2 -D__const__= -pipe -W -Wall -Werror -fPIC -fno-omit-frame-pointer
-HDRPATHS = -I$(BRPC_PATH)/output/include $(addprefix -I, $(HDRS))
-LIBPATHS = -L$(BRPC_PATH)/output/lib $(addprefix -L, $(LIBS))
+HDRS+=$(BRPC_PATH)/output/include
+LIBS+=$(BRPC_PATH)/output/lib
+HDRPATHS = $(addprefix -I, $(HDRS))
+LIBPATHS = $(addprefix -L, $(LIBS))
+COMMA=,
+SOPATHS=$(addprefix -Wl$(COMMA)-rpath=, $(LIBS))
+
 STATIC_LINKINGS += -lbrpc
 
 SOURCES = $(wildcard *.cpp)
@@ -18,7 +23,11 @@ clean:
 
 memcache_client:$(OBJS)
 	@echo "Linking $@"
+ifneq ("$(LINK_SO)", "")
+	@$(CXX) $(LIBPATHS) $(SOPATHS) -Xlinker "-(" $^ -Xlinker "-)" $(STATIC_LINKINGS) $(DYNAMIC_LINKINGS) -o $@
+else
 	@$(CXX) $(LIBPATHS) -Xlinker "-(" $^ -Wl,-Bstatic $(STATIC_LINKINGS) -Wl,-Bdynamic -Xlinker "-)" $(DYNAMIC_LINKINGS) -o $@
+endif
 
 %.o:%.cpp
 	@echo "Compiling $@"

example/multi_threaded_echo_c++/Makefile

@@ -8,8 +8,14 @@ CXXFLAGS = -std=c++0x -g -DNDEBUG -O2 -D__const__= -pipe -W -Wall -Werror -Wno-u
 ifeq ($(NEED_GPERFTOOLS), 1)
 	CXXFLAGS += -DBRPC_ENABLE_CPU_PROFILER -DBRPC_ENABLE_HEAP_PROFILER
 endif
-HDRPATHS=-I$(BRPC_PATH)/output/include $(addprefix -I, $(HDRS))
-LIBPATHS=-L$(BRPC_PATH)/output/lib $(addprefix -L, $(LIBS))
+HDRS+=$(BRPC_PATH)/output/include
+LIBS+=$(BRPC_PATH)/output/lib
+
+HDRPATHS=$(addprefix -I, $(HDRS))
+LIBPATHS=$(addprefix -L, $(LIBS))
+COMMA=,
+SOPATHS=$(addprefix -Wl$(COMMA)-rpath=, $(LIBS))
+
 STATIC_LINKINGS+=-lbrpc
 
 CLIENT_SOURCES = client.cpp
@@ -31,11 +37,19 @@ clean:
 
 echo_client:$(PROTO_OBJS) $(CLIENT_OBJS)
 	@echo "Linking $@"
-	$(CXX) $(LIBPATHS) -Xlinker "-(" $^ -Wl,-Bstatic $(STATIC_LINKINGS) -Wl,-Bdynamic -Xlinker "-)" $(DYNAMIC_LINKINGS) -o $@
+ifneq ("$(LINK_SO)", "")
+	@$(CXX) $(LIBPATHS) $(SOPATHS) -Xlinker "-(" $^ -Xlinker "-)" $(STATIC_LINKINGS) $(DYNAMIC_LINKINGS) -o $@
+else
+	@$(CXX) $(LIBPATHS) -Xlinker "-(" $^ -Wl,-Bstatic $(STATIC_LINKINGS) -Wl,-Bdynamic -Xlinker "-)" $(DYNAMIC_LINKINGS) -o $@
+endif
 
 echo_server:$(PROTO_OBJS) $(SERVER_OBJS)
 	@echo "Linking $@"
+ifneq ("$(LINK_SO)", "")
+	@$(CXX) $(LIBPATHS) $(SOPATHS) -Xlinker "-(" $^ -Xlinker "-)" $(STATIC_LINKINGS) $(DYNAMIC_LINKINGS) -o $@
+else
 	@$(CXX) $(LIBPATHS) -Xlinker "-(" $^ -Wl,-Bstatic $(STATIC_LINKINGS) -Wl,-Bdynamic -Xlinker "-)" $(DYNAMIC_LINKINGS) -o $@
+endif
 
 %.pb.cc %.pb.h:%.proto
 	@echo "Generating $@"

example/redis_c++/Makefile

 BRPC_PATH = ../../
 include $(BRPC_PATH)/config.mk
 CXXFLAGS = -std=c++0x -g -DNDEBUG -O2 -D__const__= -pipe -W -Wall -Werror -fPIC -fno-omit-frame-pointer
-HDRPATHS = -I$(BRPC_PATH)/output/include $(addprefix -I, $(HDRS))
-LIBPATHS = -L$(BRPC_PATH)/output/lib $(addprefix -L, $(LIBS))
+HDRS+=$(BRPC_PATH)/output/include
+LIBS+=$(BRPC_PATH)/output/lib
+HDRPATHS = $(addprefix -I, $(HDRS))
+LIBPATHS = $(addprefix -L, $(LIBS))
+COMMA=,
+SOPATHS=$(addprefix -Wl$(COMMA)-rpath=, $(LIBS))
+
 DYNAMIC_LINKINGS += -lreadline -lncurses
 STATIC_LINKINGS += -lbrpc
 
@@ -22,11 +27,19 @@ clean:
 
 redis_press:$(PRESS_OBJS)
 	@echo "Linking $@"
+ifneq ("$(LINK_SO)", "")
+	@$(CXX) $(LIBPATHS) $(SOPATHS) -Xlinker "-(" $^ -Xlinker "-)" $(STATIC_LINKINGS) $(DYNAMIC_LINKINGS) -o $@
+else
 	@$(CXX) $(LIBPATHS) -Xlinker "-(" $^ -Wl,-Bstatic $(STATIC_LINKINGS) -Wl,-Bdynamic -Xlinker "-)" $(DYNAMIC_LINKINGS) -o $@
+endif
 
 redis_cli:$(CLI_OBJS)
 	@echo "Linking $@"
+ifneq ("$(LINK_SO)", "")
+	@$(CXX) $(LIBPATHS) $(SOPATHS) -Xlinker "-(" $^ -Xlinker "-)" $(STATIC_LINKINGS) $(DYNAMIC_LINKINGS) -o $@
+else
 	@$(CXX) $(LIBPATHS) -Xlinker "-(" $^ -Wl,-Bstatic $(STATIC_LINKINGS) -Wl,-Bdynamic -Xlinker "-)" $(DYNAMIC_LINKINGS) -o $@
+endif
 
 %.o:%.cpp
 	@echo "Compiling $@"

src/brpc/builtin/hotspots_service.cpp

@@ -24,15 +24,13 @@
 #include "brpc/reloadable_flags.h"
 #include "brpc/builtin/pprof_perl.h"
 #include "brpc/builtin/hotspots_service.h"
+#include "brpc/details/tcmalloc_extension.h"
 
 extern "C" {
 int __attribute__((weak)) ProfilerStart(const char* fname);
 void __attribute__((weak)) ProfilerStop();
 }
 
-void __attribute__((weak)) TCMallocGetHeapSample(std::string* writer);
-void __attribute__((weak)) TCMallocGetHeapGrowthStacks(std::string* writer);
-
 namespace bthread {
 bool ContentionProfilerStart(const char* filename);
 void ContentionProfilerStop();
@@ -661,7 +659,8 @@ static void DoProfiling(ProfilingType type,
         }
         bthread::ContentionProfilerStop();
     } else if (type == PROFILING_HEAP) {
-        if ((void*)TCMallocGetHeapSample == NULL) {
+        MallocExtension* malloc_ext = MallocExtension::instance();
+        if (malloc_ext == NULL) {
             os << "Heap profiler is not enabled"
                << (use_html ? "</body></html>" : "\n");
             os.move_to(resp);
@@ -669,7 +668,7 @@ static void DoProfiling(ProfilingType type,
             return NotifyWaiters(type, cntl, view);
         }
         std::string obj;
-        TCMallocGetHeapSample(&obj);
+        malloc_ext->GetHeapSample(&obj);
         if (!WriteSmallFile(prof_name, obj)) {
             os << "Fail to write " << prof_name
                << (use_html ? "</body></html>" : "\n");
@@ -679,7 +678,8 @@ static void DoProfiling(ProfilingType type,
             return NotifyWaiters(type, cntl, view);
         }
     } else if (type == PROFILING_GROWTH) {
-        if ((void*)TCMallocGetHeapGrowthStacks == NULL) {
+        MallocExtension* malloc_ext = MallocExtension::instance();
+        if (malloc_ext == NULL) {
             os << "Growth profiler is not enabled"
                << (use_html ? "</body></html>" : "\n");
             os.move_to(resp);
@@ -687,7 +687,7 @@ static void DoProfiling(ProfilingType type,
             return NotifyWaiters(type, cntl, view);
         }
         std::string obj;
-        TCMallocGetHeapGrowthStacks(&obj);
+        malloc_ext->GetHeapGrowthStacks(&obj);
         if (!WriteSmallFile(prof_name, obj)) {
             os << "Fail to write " << prof_name
                << (use_html ? "</body></html>" : "\n");

src/brpc/builtin/pprof_service.cpp

@@ -28,6 +28,7 @@
 #include "brpc/closure_guard.h"             // ClosureGuard
 #include "brpc/builtin/pprof_service.h"
 #include "brpc/builtin/common.h"
+#include "brpc/details/tcmalloc_extension.h"
 #include "bthread/bthread.h"                // bthread_usleep
 #include "butil/fd_guard.h"
 
@@ -37,9 +38,6 @@ int __attribute__((weak)) ProfilerStart(const char* fname);
 void __attribute__((weak)) ProfilerStop();
 }
 
-void __attribute__((weak)) TCMallocGetHeapSample(std::string* writer);
-void __attribute__((weak)) TCMallocGetHeapGrowthStacks(std::string* writer);
-
 namespace bthread {
 bool ContentionProfilerStart(const char* filename);
 void ContentionProfilerStop();
@@ -204,7 +202,8 @@ void PProfService::heap(
     ::google::protobuf::Closure* done) {
     ClosureGuard done_guard(done);
     Controller* cntl = static_cast<Controller*>(controller_base);
-    if ((void*)TCMallocGetHeapSample == NULL) {
+    MallocExtension* malloc_ext = MallocExtension::instance();
+    if (malloc_ext == NULL) {
         cntl->SetFailed(ENOMETHOD, "%s, to enable heap profiler, check out "
                         "docs/cn/heap_profiler.md",
                         berror(ENOMETHOD));
@@ -220,7 +219,7 @@ void PProfService::heap(
     LOG(INFO) << " requests for heap profile";
 
     std::string obj;
-    TCMallocGetHeapSample(&obj);
+    malloc_ext->GetHeapSample(&obj);
     cntl->http_response().set_content_type("text/plain");
     cntl->response_attachment().append(obj);    
 }
@@ -232,7 +231,8 @@ void PProfService::growth(
     ::google::protobuf::Closure* done) {
     ClosureGuard done_guard(done);
     Controller* cntl = static_cast<Controller*>(controller_base);
-    if ((void*)TCMallocGetHeapGrowthStacks == NULL) {
+    MallocExtension* malloc_ext = MallocExtension::instance();
+    if (malloc_ext == NULL) {
         cntl->SetFailed(ENOMETHOD, "%s, to enable growth profiler, check out "
                         "docs/cn/heap_profiler.md",
                         berror(ENOMETHOD));
@@ -248,7 +248,7 @@ void PProfService::growth(
     LOG(INFO) << " requests for growth profile";
 
     std::string obj;
-    TCMallocGetHeapGrowthStacks(&obj);
+    malloc_ext->GetHeapGrowthStacks(&obj);
     cntl->http_response().set_content_type("text/plain");
     cntl->response_attachment().append(obj);    
 }

src/brpc/details/profiler_linker.h

@@ -30,13 +30,6 @@ void ProfilerStop(const char*);
 #endif
 #endif
 
-#if defined(BRPC_ENABLE_HEAP_PROFILER) || defined(BAIDU_RPC_ENABLE_HEAP_PROFILER)
-#include <string>
-void TCMallocGetHeapSample(std::string* writer);
-void TCMallocGetHeapGrowthStacks(std::string* writer);
-#endif
-
-
 namespace brpc {
 
 // defined in src/brpc/builtin/index_service.cpp
@@ -49,7 +42,7 @@ extern int PROFILER_LINKER_DUMMY;
 struct ProfilerLinker {
     // [ Must be inlined ]
     // This function is included by user's compilation unit to force
-    // linking of ProfilerStart()/ProfilerStop()/TCMallocGetHeapSample()
+    // linking of ProfilerStart()/ProfilerStop()
     // etc when corresponding macros are defined.
     inline ProfilerLinker() {
         
@@ -64,10 +57,6 @@ struct ProfilerLinker {
     
 #if defined(BRPC_ENABLE_HEAP_PROFILER) || defined(BAIDU_RPC_ENABLE_HEAP_PROFILER)
         heap_profiler_enabled = true;
-        if (PROFILER_LINKER_DUMMY != 0/*must be false*/) {
-            TCMallocGetHeapSample(NULL);
-            TCMallocGetHeapGrowthStacks(NULL);
-        }
 #endif
     }
 };

src/brpc/details/tcmalloc_extension.cpp

+#include "butil/compiler_specific.h"
+#include "brpc/details/tcmalloc_extension.h"
+
+MallocExtension* BAIDU_WEAK MallocExtension::instance() {
+    return NULL;
+}

src/brpc/tcmalloc_extension.cpp → src/brpc/details/tcmalloc_extension.h

-// This file pastes google/malloc_extension.h and implements
-// TCMallocGetHeapSample in the last sections.
-
 // Copyright (c) 2005, Google Inc.
 // All rights reserved.
 // 
@@ -65,7 +62,7 @@ static const int kMallocHistogramSize = 64;
 
 typedef std::string MallocExtensionWriter;
 
-namespace butil {
+namespace base {
 struct MallocRange;
 }
 
@@ -120,7 +117,7 @@ class PERFTOOLS_DLL_DECL MallocExtension {
   //
   // This is a best-effort interface useful only for performance
   // analysis.  The implementation may not call func at all.
-  typedef void (RangeFunction)(void*, const butil::MallocRange*);
+  typedef void (RangeFunction)(void*, const base::MallocRange*);
   virtual void Ranges(void* arg, RangeFunction func);
 
   // -------------------------------------------------------------------
@@ -314,12 +311,3 @@ class PERFTOOLS_DLL_DECL MallocExtension {
   // in the address space size.
   virtual void** ReadHeapGrowthStackTraces();
 };
-
-// Following functions will not be linked when BRPC_ENABLE_HEAP_PROFILER is
-// undefined.
-void TCMallocGetHeapSample(std::string* writer) {
-    MallocExtension::instance()->GetHeapSample(writer);
-}
-void TCMallocGetHeapGrowthStacks(std::string* writer) {
-    MallocExtension::instance()->GetHeapGrowthStacks(writer);
-}