From 8142fedd196e20d24c48485396821b12034e7eb9 Mon Sep 17 00:00:00 2001 From: mustiikhalil Date: Fri, 26 Feb 2021 01:38:12 +0300 Subject: Working on a python example plus fixing python grpc code (#6456) Refactored python grpc code gen Adds example server & client + fixes ci Fixes generated code Making sure we encode the reply string as utf8 Adds Readme details to clarify issue regarding encoding when python is sending/receiving --- CMakeLists.txt | 1 - grpc/examples/README.md | 28 + grpc/examples/generate.sh | 19 +- grpc/examples/python/greeter/README.md | 12 + grpc/examples/python/greeter/client.py | 40 ++ grpc/examples/python/greeter/models/HelloReply.py | 45 ++ .../examples/python/greeter/models/HelloRequest.py | 45 ++ grpc/examples/python/greeter/models/__init__.py | 0 .../python/greeter/models/greeter_grpc_fb.py | 52 ++ grpc/examples/python/greeter/server.py | 57 ++ grpc/src/compiler/BUILD | 1 - grpc/src/compiler/python_generator.cc | 683 ++++----------------- grpc/src/compiler/python_generator.h | 13 +- grpc/src/compiler/python_private_generator.h | 72 --- src/idl_gen_grpc.cpp | 63 +- 15 files changed, 441 insertions(+), 690 deletions(-) create mode 100644 grpc/examples/python/greeter/README.md create mode 100644 grpc/examples/python/greeter/client.py create mode 100644 grpc/examples/python/greeter/models/HelloReply.py create mode 100644 grpc/examples/python/greeter/models/HelloRequest.py create mode 100644 grpc/examples/python/greeter/models/__init__.py create mode 100644 grpc/examples/python/greeter/models/greeter_grpc_fb.py create mode 100644 grpc/examples/python/greeter/server.py delete mode 100644 grpc/src/compiler/python_private_generator.h diff --git a/CMakeLists.txt b/CMakeLists.txt index f56f844f..ba14f7c2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -120,7 +120,6 @@ set(FlatBuffers_Compiler_SRCS grpc/src/compiler/java_generator.h grpc/src/compiler/java_generator.cc grpc/src/compiler/python_generator.h - grpc/src/compiler/python_private_generator.h grpc/src/compiler/python_generator.cc grpc/src/compiler/swift_generator.h grpc/src/compiler/swift_generator.cc diff --git a/grpc/examples/README.md b/grpc/examples/README.md index f5694a84..c821218b 100644 --- a/grpc/examples/README.md +++ b/grpc/examples/README.md @@ -1,5 +1,33 @@ ## Languages known issues +### Python + +- Assert the type required in your server/client since python is able to receive `Bytes array` or `utf8 strings`. + +```python +def SayHello(self, request, context): + # request might be a byte array or a utf8 string + + r = HelloRequest.HelloRequest().GetRootAs(request, 0) + reply = "Unknown" + if r.Name(): + reply = r.Name() + # Issues might happen if type checking isnt present. + # thus encoding it as a `reply.decode('UTF-8')` + return build_reply("welcome " + reply.decode('UTF-8')) + +``` + +This can be prevented by making sure all the requests coming to/from python are `Bytes array` + +```python +def say_hello(stub, builder): + hello_request = bytes(builder.Output()) + reply = stub.SayHello(hello_request) + r = HelloReply.HelloReply.GetRootAs(reply) + print(r.Message()) +``` + ### Go - Always requires the `content-type` of the payload to be set to `application/grpc+flatbuffers` diff --git a/grpc/examples/generate.sh b/grpc/examples/generate.sh index 0ac2818a..b0740d8a 100644 --- a/grpc/examples/generate.sh +++ b/grpc/examples/generate.sh @@ -38,23 +38,34 @@ fi generator="--grpc $current_dir/greeter.fbs" # Regenerate Go lang code -cd go/ +cd go cd greeter fbc --go ${generator} cd ${current_dir} -cd swift/ +# Regenerate Python code +cd python + +cd greeter + +fbc --python ${generator} + +cd ${current_dir} + +# Regenerate Swift code +cd swift cd Greeter/Sources/Model fbc --swift ${generator} cd ${current_dir} -cd ts/ +# Regenerate Typescript code +cd ts cd greeter/src fbc --ts ${generator} -cd ${current_dir} \ No newline at end of file +cd ${current_dir} diff --git a/grpc/examples/python/greeter/README.md b/grpc/examples/python/greeter/README.md new file mode 100644 index 00000000..fcf310c0 --- /dev/null +++ b/grpc/examples/python/greeter/README.md @@ -0,0 +1,12 @@ +# Python Greeter example + +## Prerequisite + +- You need to have grpc python installed on your device `pip install grpcio` +## How to run Server: + +- `python server.py ${PORT}` + +## How to run Client: + +- `python client.py ${PORT} ${NAME}` \ No newline at end of file diff --git a/grpc/examples/python/greeter/client.py b/grpc/examples/python/greeter/client.py new file mode 100644 index 00000000..d2d71845 --- /dev/null +++ b/grpc/examples/python/greeter/client.py @@ -0,0 +1,40 @@ +import sys +import argparse +import grpc + +sys.path.insert(0, '../../../../../flatbuffers/python') + +import flatbuffers +from models import HelloReply, HelloRequest, greeter_grpc_fb + +parser = argparse.ArgumentParser() +parser.add_argument("port", help="server port to connect to", default=3000) +parser.add_argument("name", help="name to be sent to server", default="flatbuffers") + +def say_hello(stub, hello_request): + reply = stub.SayHello(hello_request) + r = HelloReply.HelloReply.GetRootAs(reply) + print(r.Message()) + +def say_many_hellos(stub, hello_request): + greetings = stub.SayManyHellos(hello_request) + for greeting in greetings: + r = HelloReply.HelloReply.GetRootAs(greeting) + print(r.Message()) + +def main(): + args = parser.parse_args() + + with grpc.insecure_channel('localhost:' + args.port) as channel: + builder = flatbuffers.Builder() + ind = builder.CreateString(args.name) + HelloRequest.HelloRequestStart(builder) + HelloRequest.HelloRequestAddName(builder, ind) + root = HelloRequest.HelloRequestEnd(builder) + builder.Finish(root) + output = bytes(builder.Output()) + stub = greeter_grpc_fb.GreeterStub(channel) + say_hello(stub, output) + say_many_hellos(stub, output) + +main() \ No newline at end of file diff --git a/grpc/examples/python/greeter/models/HelloReply.py b/grpc/examples/python/greeter/models/HelloReply.py new file mode 100644 index 00000000..95434dcc --- /dev/null +++ b/grpc/examples/python/greeter/models/HelloReply.py @@ -0,0 +1,45 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# namespace: models + +import flatbuffers +from flatbuffers.compat import import_numpy +np = import_numpy() + +class HelloReply(object): + __slots__ = ['_tab'] + + @classmethod + def GetRootAs(cls, buf, offset=0): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) + x = HelloReply() + x.Init(buf, n + offset) + return x + + @classmethod + def GetRootAsHelloReply(cls, buf, offset=0): + """This method is deprecated. Please switch to GetRootAs.""" + return cls.GetRootAs(buf, offset) + # HelloReply + def Init(self, buf, pos): + self._tab = flatbuffers.table.Table(buf, pos) + + # HelloReply + def Message(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + return self._tab.String(o + self._tab.Pos) + return None + +def Start(builder): builder.StartObject(1) +def HelloReplyStart(builder): + """This method is deprecated. Please switch to Start.""" + return Start(builder) +def AddMessage(builder, message): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(message), 0) +def HelloReplyAddMessage(builder, message): + """This method is deprecated. Please switch to AddMessage.""" + return AddMessage(builder, message) +def End(builder): return builder.EndObject() +def HelloReplyEnd(builder): + """This method is deprecated. Please switch to End.""" + return End(builder) \ No newline at end of file diff --git a/grpc/examples/python/greeter/models/HelloRequest.py b/grpc/examples/python/greeter/models/HelloRequest.py new file mode 100644 index 00000000..0263095e --- /dev/null +++ b/grpc/examples/python/greeter/models/HelloRequest.py @@ -0,0 +1,45 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# namespace: models + +import flatbuffers +from flatbuffers.compat import import_numpy +np = import_numpy() + +class HelloRequest(object): + __slots__ = ['_tab'] + + @classmethod + def GetRootAs(cls, buf, offset=0): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) + x = HelloRequest() + x.Init(buf, n + offset) + return x + + @classmethod + def GetRootAsHelloRequest(cls, buf, offset=0): + """This method is deprecated. Please switch to GetRootAs.""" + return cls.GetRootAs(buf, offset) + # HelloRequest + def Init(self, buf, pos): + self._tab = flatbuffers.table.Table(buf, pos) + + # HelloRequest + def Name(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + return self._tab.String(o + self._tab.Pos) + return None + +def Start(builder): builder.StartObject(1) +def HelloRequestStart(builder): + """This method is deprecated. Please switch to Start.""" + return Start(builder) +def AddName(builder, name): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(name), 0) +def HelloRequestAddName(builder, name): + """This method is deprecated. Please switch to AddName.""" + return AddName(builder, name) +def End(builder): return builder.EndObject() +def HelloRequestEnd(builder): + """This method is deprecated. Please switch to End.""" + return End(builder) \ No newline at end of file diff --git a/grpc/examples/python/greeter/models/__init__.py b/grpc/examples/python/greeter/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/grpc/examples/python/greeter/models/greeter_grpc_fb.py b/grpc/examples/python/greeter/models/greeter_grpc_fb.py new file mode 100644 index 00000000..b9f8a4ed --- /dev/null +++ b/grpc/examples/python/greeter/models/greeter_grpc_fb.py @@ -0,0 +1,52 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! + +import grpc + +class GreeterStub(object): + """ Interface exported by the server. """ + + def __init__(self, channel): + """ Constructor. + + Args: + channel: A grpc.Channel. + """ + + self.SayHello = channel.unary_unary( + "/models.Greeter/SayHello" + ) + + self.SayManyHellos = channel.unary_stream( + "/models.Greeter/SayManyHellos" + ) + + +class GreeterServicer(object): + """ Interface exported by the server. """ + + def SayHello(self, request, context): + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + + def SayManyHellos(self, request, context): + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + + +def add_GreeterServicer_to_server(servicer, server): + rpc_method_handlers = { + 'SayHello': grpc.unary_unary_rpc_method_handler( + servicer.SayHello + ), + 'SayManyHellos': grpc.unary_stream_rpc_method_handler( + servicer.SayManyHellos + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'models.Greeter', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + diff --git a/grpc/examples/python/greeter/server.py b/grpc/examples/python/greeter/server.py new file mode 100644 index 00000000..acca880b --- /dev/null +++ b/grpc/examples/python/greeter/server.py @@ -0,0 +1,57 @@ +from concurrent import futures +import sys +import argparse +import grpc + +sys.path.insert(0, '../../../../../flatbuffers/python') + +import flatbuffers +from models import HelloReply, HelloRequest, greeter_grpc_fb + +parser = argparse.ArgumentParser() +parser.add_argument("port", help="server on port", default=3000) + +def build_reply(message): + builder = flatbuffers.Builder() + ind = builder.CreateString(message) + HelloReply.HelloReplyStart(builder) + HelloReply.HelloReplyAddMessage(builder, ind) + root = HelloReply.HelloReplyEnd(builder) + builder.Finish(root) + return bytes(builder.Output()) + +class GreeterServicer(greeter_grpc_fb.GreeterServicer): + + def __init__(self): + self.greetings = ["Hi", "Hallo", "Ciao"] + + def SayHello(self, request, context): + r = HelloRequest.HelloRequest().GetRootAs(request, 0) + reply = "Unknown" + if r.Name(): + reply = r.Name() + return build_reply("welcome " + reply.decode('UTF-8')) + + def SayManyHellos(self, request, context): + r = HelloRequest.HelloRequest().GetRootAs(request, 0) + reply = "Unknown" + if r.Name(): + reply = r.Name() + + for greeting in self.greetings: + print(type(reply)) + yield build_reply(greeting + " " + reply.decode('UTF-8')) + + +def serve(): + args = parser.parse_args() + server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + greeter_grpc_fb.add_GreeterServicer_to_server( + GreeterServicer(), server + ) + server.add_insecure_port('[::]:' + args.port) + server.start() + server.wait_for_termination() + +if __name__ == '__main__': + serve() \ No newline at end of file diff --git a/grpc/src/compiler/BUILD b/grpc/src/compiler/BUILD index 23fe5405..88e27e80 100644 --- a/grpc/src/compiler/BUILD +++ b/grpc/src/compiler/BUILD @@ -79,7 +79,6 @@ cc_library( ], hdrs = [ "python_generator.h", - "python_private_generator.h", ":common_headers", ], include_prefix = "src/compiler", diff --git a/grpc/src/compiler/python_generator.cc b/grpc/src/compiler/python_generator.cc index 3fcf7ea4..8108db45 100644 --- a/grpc/src/compiler/python_generator.cc +++ b/grpc/src/compiler/python_generator.cc @@ -16,608 +16,133 @@ * */ -#include -#include -#include -#include -#include -#include #include -#include -#include -#include #include -#include -#include #include "flatbuffers/util.h" #include "src/compiler/python_generator.h" -#include "src/compiler/python_private_generator.h" - -using std::make_pair; -using std::map; -using std::pair; -using std::replace; -using std::tuple; -using std::vector; -using std::set; namespace grpc_python_generator { -grpc::string generator_file_name; - -typedef map StringMap; -typedef vector StringVector; -typedef tuple StringPair; -typedef set StringPairSet; - -// Provides RAII indentation handling. Use as: -// { -// IndentScope raii_my_indent_var_name_here(my_py_printer); -// // constructor indented my_py_printer -// ... -// // destructor called at end of scope, un-indenting my_py_printer -// } -class IndentScope { - public: - explicit IndentScope(grpc_generator::Printer* printer) : printer_(printer) { - printer_->Indent(); - } - - ~IndentScope() { printer_->Outdent(); } - - private: - grpc_generator::Printer* printer_; -}; +grpc::string GenerateMethodType(const grpc_generator::Method *method) { -inline grpc::string StringReplace(grpc::string str, const grpc::string& from, - const grpc::string& to, bool replace_all) { - size_t pos = 0; + if (method->NoStreaming()) + return "unary_unary"; - do { - pos = str.find(from, pos); - if (pos == grpc::string::npos) { - break; - } - str.replace(pos, from.length(), to); - pos += to.length(); - } while (replace_all); + if (method->ServerStreaming()) + return "unary_stream"; - return str; -} - -inline grpc::string StringReplace(grpc::string str, const grpc::string& from, - const grpc::string& to) { - return StringReplace(str, from, to, true); -} + if (method->ClientStreaming()) + return "stream_unary"; -grpc::string ModuleName(const grpc::string& filename, - const grpc::string& import_prefix) { - grpc::string basename = flatbuffers::StripExtension(filename); - basename = StringReplace(basename, "-", "_"); - basename = StringReplace(basename, "/", "."); - return import_prefix + basename + "_fb"; + return "stream_stream"; } -grpc::string ModuleAlias(const grpc::string& filename, - const grpc::string& import_prefix) { - grpc::string module_name = ModuleName(filename, import_prefix); - // We can't have dots in the module name, so we replace each with _dot_. - // But that could lead to a collision between a.b and a_dot_b, so we also - // duplicate each underscore. - module_name = StringReplace(module_name, "_", "__"); - module_name = StringReplace(module_name, ".", "_dot_"); - return module_name; -} - -PrivateGenerator::PrivateGenerator(const GeneratorConfiguration& config_, - const grpc_generator::File* file_) - : config(config_), file(file_) {} - -void PrivateGenerator::PrintBetaServicer(const grpc_generator::Service* service, - grpc_generator::Printer* out) { - StringMap service_dict; - service_dict["Service"] = service->name(); - out->Print("\n\n"); - out->Print(service_dict, "class Beta$Service$Servicer(object):\n"); - { - IndentScope raii_class_indent(out); - out->Print( - "\"\"\"The Beta API is deprecated for 0.15.0 and later.\n" - "\nIt is recommended to use the GA API (classes and functions in this\n" - "file not marked beta) for all further purposes. This class was " - "generated\n" - "only to ease transition from grpcio<0.15.0 to " - "grpcio>=0.15.0.\"\"\"\n"); - for (int i = 0; i < service->method_count(); ++i) { - auto method = service->method(i); - grpc::string arg_name = - method->ClientStreaming() ? "request_iterator" : "request"; - StringMap method_dict; - method_dict["Method"] = method->name(); - method_dict["ArgName"] = arg_name; - out->Print(method_dict, "def $Method$(self, $ArgName$, context):\n"); - { - IndentScope raii_method_indent(out); - out->Print("context.code(beta_interfaces.StatusCode.UNIMPLEMENTED)\n"); - } - } - } -} +grpc::string GenerateMethodInput(const grpc_generator::Method *method) { -void PrivateGenerator::PrintBetaStub(const grpc_generator::Service* service, - grpc_generator::Printer* out) { - StringMap service_dict; - service_dict["Service"] = service->name(); - out->Print("\n\n"); - out->Print(service_dict, "class Beta$Service$Stub(object):\n"); - { - IndentScope raii_class_indent(out); - out->Print( - "\"\"\"The Beta API is deprecated for 0.15.0 and later.\n" - "\nIt is recommended to use the GA API (classes and functions in this\n" - "file not marked beta) for all further purposes. This class was " - "generated\n" - "only to ease transition from grpcio<0.15.0 to " - "grpcio>=0.15.0.\"\"\"\n"); - for (int i = 0; i < service->method_count(); ++i) { - auto method = service->method(i); - grpc::string arg_name = - method->ClientStreaming() ? "request_iterator" : "request"; - StringMap method_dict; - method_dict["Method"] = method->name(); - method_dict["ArgName"] = arg_name; - out->Print(method_dict, - "def $Method$(self, $ArgName$, timeout, metadata=None, " - "with_call=False, protocol_options=None):\n"); - { - IndentScope raii_method_indent(out); - out->Print("raise NotImplementedError()\n"); - } - if (!method->ServerStreaming()) { - out->Print(method_dict, "$Method$.future = None\n"); - } - } - } -} + if (method->NoStreaming() || method->ServerStreaming()) + return "self, request, context"; -void PrivateGenerator::PrintBetaServerFactory( - const grpc::string& package_qualified_service_name, - const grpc_generator::Service* service, grpc_generator::Printer* out) { - StringMap service_dict; - service_dict["Service"] = service->name(); - out->Print("\n\n"); - out->Print(service_dict, - "def beta_create_$Service$_server(servicer, pool=None, " - "pool_size=None, default_timeout=None, maximum_timeout=None):\n"); - { - IndentScope raii_create_server_indent(out); - out->Print( - "\"\"\"The Beta API is deprecated for 0.15.0 and later.\n" - "\nIt is recommended to use the GA API (classes and functions in this\n" - "file not marked beta) for all further purposes. This function was\n" - "generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0" - "\"\"\"\n"); - StringMap method_implementation_constructors; - StringMap input_message_modules_and_classes; - StringMap output_message_modules_and_classes; - for (int i = 0; i < service->method_count(); ++i) { - auto method = service->method(i); - const grpc::string method_implementation_constructor = - grpc::string(method->ClientStreaming() ? "stream_" : "unary_") + - grpc::string(method->ServerStreaming() ? "stream_" : "unary_") + - "inline"; - grpc::string input_message_module_and_class = method->get_fb_builder(); - grpc::string output_message_module_and_class = method->get_fb_builder(); - method_implementation_constructors.insert( - make_pair(method->name(), method_implementation_constructor)); - input_message_modules_and_classes.insert( - make_pair(method->name(), input_message_module_and_class)); - output_message_modules_and_classes.insert( - make_pair(method->name(), output_message_module_and_class)); - } - StringMap method_dict; - method_dict["PackageQualifiedServiceName"] = package_qualified_service_name; -// out->Print("request_deserializers = {\n"); -// for (StringMap::iterator name_and_input_module_class_pair = -// input_message_modules_and_classes.begin(); -// name_and_input_module_class_pair != -// input_message_modules_and_classes.end(); -// name_and_input_module_class_pair++) { -// method_dict["MethodName"] = name_and_input_module_class_pair->first; -// method_dict["InputTypeModuleAndClass"] = -// name_and_input_module_class_pair->second; -// IndentScope raii_indent(out); -// out->Print(method_dict, -// "(\'$PackageQualifiedServiceName$\', \'$MethodName$\'): " -// "$InputTypeModuleAndClass$.FromString,\n"); -// } -// out->Print("}\n"); -// out->Print("response_serializers = {\n"); -// for (StringMap::iterator name_and_output_module_class_pair = -// output_message_modules_and_classes.begin(); -// name_and_output_module_class_pair != -// output_message_modules_and_classes.end(); -// name_and_output_module_class_pair++) { -// method_dict["MethodName"] = name_and_output_module_class_pair->first; -// method_dict["OutputTypeModuleAndClass"] = -// name_and_output_module_class_pair->second; -// IndentScope raii_indent(out); -// out->Print(method_dict, -// "(\'$PackageQualifiedServiceName$\', \'$MethodName$\'): " -// "$OutputTypeModuleAndClass$.SerializeToString,\n"); -// } -// out->Print("}\n"); - out->Print("method_implementations = {\n"); - for (StringMap::iterator name_and_implementation_constructor = - method_implementation_constructors.begin(); - name_and_implementation_constructor != - method_implementation_constructors.end(); - name_and_implementation_constructor++) { - method_dict["Method"] = name_and_implementation_constructor->first; - method_dict["Constructor"] = name_and_implementation_constructor->second; - IndentScope raii_descriptions_indent(out); - const grpc::string method_name = - name_and_implementation_constructor->first; - out->Print(method_dict, - "(\'$PackageQualifiedServiceName$\', \'$Method$\'): " - "face_utilities.$Constructor$(servicer.$Method$),\n"); - } - out->Print("}\n"); - out->Print( - "server_options = beta_implementations.server_options(" - "thread_pool=pool, thread_pool_size=pool_size, " - "default_timeout=default_timeout, " - "maximum_timeout=maximum_timeout)\n"); - out->Print( - "return beta_implementations.server(method_implementations, " - "options=server_options)\n"); - //"request_deserializers=request_deserializers, " - //"response_serializers=response_serializers, " - } + return "self, request_iterator, context"; } -void PrivateGenerator::PrintBetaStubFactory( - const grpc::string& package_qualified_service_name, - const grpc_generator::Service* service, grpc_generator::Printer* out) { - StringMap dict; - dict["Service"] = service->name(); - out->Print("\n\n"); - out->Print(dict, - "def beta_create_$Service$_stub(channel, host=None," - " metadata_transformer=None, pool=None, pool_size=None):\n"); - { - IndentScope raii_create_server_indent(out); - out->Print( - "\"\"\"The Beta API is deprecated for 0.15.0 and later.\n" - "\nIt is recommended to use the GA API (classes and functions in this\n" - "file not marked beta) for all further purposes. This function was\n" - "generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0" - "\"\"\"\n"); - StringMap method_cardinalities; - StringMap input_message_modules_and_classes; - StringMap output_message_modules_and_classes; - for (int i = 0; i < service->method_count(); ++i) { - auto method = service->method(i); - const grpc::string method_cardinality = - grpc::string(method->ClientStreaming() ? "STREAM" : "UNARY") + - "_" + - grpc::string(method->ServerStreaming() ? "STREAM" : "UNARY"); - grpc::string input_message_module_and_class = method->get_fb_builder(); - grpc::string output_message_module_and_class = method->get_fb_builder(); - method_cardinalities.insert( - make_pair(method->name(), method_cardinality)); - input_message_modules_and_classes.insert( - make_pair(method->name(), input_message_module_and_class)); - output_message_modules_and_classes.insert( - make_pair(method->name(), output_message_module_and_class)); - } - StringMap method_dict; - method_dict["PackageQualifiedServiceName"] = package_qualified_service_name; -// out->Print("request_serializers = {\n"); -// for (StringMap::iterator name_and_input_module_class_pair = -// input_message_modules_and_classes.begin(); -// name_and_input_module_class_pair != -// input_message_modules_and_classes.end(); -// name_and_input_module_class_pair++) { -// method_dict["MethodName"] = name_and_input_module_class_pair->first; -// method_dict["InputTypeModuleAndClass"] = -// name_and_input_module_class_pair->second; -// IndentScope raii_indent(out); -// out->Print(method_dict, -// "(\'$PackageQualifiedServiceName$\', \'$MethodName$\'): " -// "$InputTypeModuleAndClass$.SerializeToString,\n"); -// } -// out->Print("}\n"); -// out->Print("response_deserializers = {\n"); -// for (StringMap::iterator name_and_output_module_class_pair = -// output_message_modules_and_classes.begin(); -// name_and_output_module_class_pair != -// output_message_modules_and_classes.end(); -// name_and_output_module_class_pair++) { -// method_dict["MethodName"] = name_and_output_module_class_pair->first; -// method_dict["OutputTypeModuleAndClass"] = -// name_and_output_module_class_pair->second; -// IndentScope raii_indent(out); -// out->Print(method_dict, -// "(\'$PackageQualifiedServiceName$\', \'$MethodName$\'): " -// "$OutputTypeModuleAndClass$.FromString,\n"); -// } -// out->Print("}\n"); - out->Print("cardinalities = {\n"); - for (StringMap::iterator name_and_cardinality = - method_cardinalities.begin(); - name_and_cardinality != method_cardinalities.end(); - name_and_cardinality++) { - method_dict["Method"] = name_and_cardinality->first; - method_dict["Cardinality"] = name_and_cardinality->second; - IndentScope raii_descriptions_indent(out); - out->Print(method_dict, - "\'$Method$\': cardinality.Cardinality.$Cardinality$,\n"); - } - out->Print("}\n"); - out->Print( - "stub_options = beta_implementations.stub_options(" - "host=host, metadata_transformer=metadata_transformer, " - "thread_pool=pool, thread_pool_size=pool_size)\n"); - out->Print(method_dict, - "return beta_implementations.dynamic_stub(channel, " - "\'$PackageQualifiedServiceName$\', " - "cardinalities, options=stub_options)\n"); - // "request_serializers=request_serializers, " - //"response_deserializers=response_deserializers, " - } -} +void GenerateStub(const grpc_generator::Service *service, + grpc_generator::Printer *printer, + std::map *dictonary) { + auto vars = *dictonary; + printer->Print(vars, "class $ServiceName$Stub(object):\n"); + printer->Indent(); + printer->Print("\"\"\" Interface exported by the server. \"\"\""); + printer->Print("\n\n"); + printer->Print("def __init__(self, channel):\n"); + printer->Indent(); + printer->Print("\"\"\" Constructor. \n\n"); + printer->Print("Args: \n"); + printer->Print("channel: A grpc.Channel. \n"); + printer->Print("\"\"\"\n\n"); -void PrivateGenerator::PrintStub( - const grpc::string& package_qualified_service_name, - const grpc_generator::Service* service, grpc_generator::Printer* out) { - StringMap dict; - dict["Service"] = service->name(); - out->Print("\n\n"); - out->Print(dict, "class $Service$Stub(object):\n"); - { - IndentScope raii_class_indent(out); - out->Print("\n"); - out->Print("def __init__(self, channel):\n"); - { - IndentScope raii_init_indent(out); - out->Print("\"\"\"Constructor.\n"); - out->Print("\n"); - out->Print("Args:\n"); - { - IndentScope raii_args_indent(out); - out->Print("channel: A grpc.Channel.\n"); - } - out->Print("\"\"\"\n"); - for (int i = 0; i < service->method_count(); ++i) { - auto method = service->method(i); - grpc::string multi_callable_constructor = - grpc::string(method->ClientStreaming() ? "stream" : "unary") + - "_" + - grpc::string(method->ServerStreaming() ? "stream" : "unary"); - grpc::string request_module_and_class = method->get_fb_builder(); - grpc::string response_module_and_class = method->get_fb_builder(); - StringMap method_dict; - method_dict["Method"] = method->name(); - method_dict["MultiCallableConstructor"] = multi_callable_constructor; - out->Print(method_dict, - "self.$Method$ = channel.$MultiCallableConstructor$(\n"); - { - method_dict["PackageQualifiedService"] = - package_qualified_service_name; - method_dict["RequestModuleAndClass"] = request_module_and_class; - method_dict["ResponseModuleAndClass"] = response_module_and_class; - IndentScope raii_first_attribute_indent(out); - IndentScope raii_second_attribute_indent(out); - out->Print(method_dict, "'/$PackageQualifiedService$/$Method$',\n"); - out->Print(method_dict,"\n"); - out->Print( - method_dict,"\n"); - out->Print(")\n"); - } - } - } + for (int j = 0; j < service->method_count(); j++) { + auto method = service->method(j); + vars["MethodName"] = method->name(); + vars["MethodType"] = GenerateMethodType(&*method); + printer->Print(vars, "self.$MethodName$ = channel.$MethodType$(\n"); + printer->Indent(); + printer->Print(vars, "\"/$PATH$$ServiceName$/$MethodName$\"\n"); + printer->Print(")\n"); + printer->Outdent(); + printer->Print("\n"); } -} - -void PrivateGenerator::PrintServicer(const grpc_generator::Service* service, - grpc_generator::Printer* out) { - StringMap service_dict; - service_dict["Service"] = service->name(); - out->Print("\n\n"); - out->Print(service_dict, "class $Service$Servicer(object):\n"); - { - IndentScope raii_class_indent(out); - for (int i = 0; i < service->method_count(); ++i) { - auto method = service->method(i); - grpc::string arg_name = - method->ClientStreaming() ? "request_iterator" : "request"; - StringMap method_dict; - method_dict["Method"] = method->name(); - method_dict["ArgName"] = arg_name; - out->Print("\n"); - out->Print(method_dict, "def $Method$(self, $ArgName$, context):\n"); - { - IndentScope raii_method_indent(out); - out->Print("context.set_code(grpc.StatusCode.UNIMPLEMENTED)\n"); - out->Print("context.set_details('Method not implemented!')\n"); - out->Print("raise NotImplementedError('Method not implemented!')\n"); - } - } + printer->Outdent(); + printer->Outdent(); + printer->Print("\n"); +} + +void GenerateServicer(const grpc_generator::Service *service, + grpc_generator::Printer *printer, + std::map *dictonary) { + auto vars = *dictonary; + printer->Print(vars, "class $ServiceName$Servicer(object):\n"); + printer->Indent(); + printer->Print("\"\"\" Interface exported by the server. \"\"\""); + printer->Print("\n\n"); + + for (int j = 0; j < service->method_count(); j++) { + auto method = service->method(j); + vars["MethodName"] = method->name(); + vars["MethodInput"] = GenerateMethodInput(&*method); + printer->Print(vars, "def $MethodName$($MethodInput$):\n"); + printer->Indent(); + printer->Print("context.set_code(grpc.StatusCode.UNIMPLEMENTED)\n"); + printer->Print("context.set_details('Method not implemented!')\n"); + printer->Print("raise NotImplementedError('Method not implemented!')\n"); + printer->Outdent(); + printer->Print("\n\n"); } -} - -void PrivateGenerator::PrintAddServicerToServer( - const grpc::string& package_qualified_service_name, - const grpc_generator::Service* service, grpc_generator::Printer* out) { - StringMap service_dict; - service_dict["Service"] = service->name(); - out->Print("\n\n"); - out->Print(service_dict, - "def add_$Service$Servicer_to_server(servicer, server):\n"); - { - IndentScope raii_class_indent(out); - out->Print("rpc_method_handlers = {\n"); - { - IndentScope raii_dict_first_indent(out); - IndentScope raii_dict_second_indent(out); - for (int i = 0; i < service->method_count(); ++i) { - auto method = service->method(i); - grpc::string method_handler_constructor = - grpc::string(method->ClientStreaming() ? "stream" : "unary") + - "_" + - grpc::string(method->ServerStreaming() ? "stream" : "unary") + - "_rpc_method_handler"; - grpc::string request_module_and_class = method->get_fb_builder(); - grpc::string response_module_and_class = method->get_fb_builder(); - StringMap method_dict; - method_dict["Method"] = method->name(); - method_dict["MethodHandlerConstructor"] = method_handler_constructor; - method_dict["RequestModuleAndClass"] = request_module_and_class; - method_dict["ResponseModuleAndClass"] = response_module_and_class; - out->Print(method_dict, - "'$Method$': grpc.$MethodHandlerConstructor$(\n"); - { - IndentScope raii_call_first_indent(out); - IndentScope raii_call_second_indent(out); - out->Print(method_dict, "servicer.$Method$,\n"); - out->Print( - method_dict,"\n"); - out->Print( - method_dict, - "\n"); - } - out->Print("),\n"); - } - } - StringMap method_dict; - method_dict["PackageQualifiedServiceName"] = package_qualified_service_name; - out->Print("}\n"); - out->Print("generic_handler = grpc.method_handlers_generic_handler(\n"); - { - IndentScope raii_call_first_indent(out); - IndentScope raii_call_second_indent(out); - out->Print(method_dict, - "'$PackageQualifiedServiceName$', rpc_method_handlers)\n"); - } - out->Print("server.add_generic_rpc_handlers((generic_handler,))\n"); + printer->Outdent(); + printer->Print("\n"); + +} + +void GenerateRegister(const grpc_generator::Service *service, + grpc_generator::Printer *printer, + std::map *dictonary) { + auto vars = *dictonary; + printer->Print(vars, "def add_$ServiceName$Servicer_to_server(servicer, server):\n"); + printer->Indent(); + printer->Print("rpc_method_handlers = {\n"); + printer->Indent(); + for (int j = 0; j < service->method_count(); j++) { + auto method = service->method(j); + vars["MethodName"] = method->name(); + vars["MethodType"] = GenerateMethodType(&*method); + printer->Print(vars, "'$MethodName$': grpc.$MethodType$_rpc_method_handler(\n"); + printer->Indent(); + printer->Print(vars, "servicer.$MethodName$\n"); + printer->Outdent(); + printer->Print("),\n"); } -} - -void PrivateGenerator::PrintBetaPreamble(grpc_generator::Printer* out) { - StringMap var; - var["Package"] = config.beta_package_root; - out->Print(var, - "from $Package$ import implementations as beta_implementations\n"); - out->Print(var, "from $Package$ import interfaces as beta_interfaces\n"); - out->Print("from grpc.framework.common import cardinality\n"); - out->Print( - "from grpc.framework.interfaces.face import utilities as " - "face_utilities\n"); -} - -void PrivateGenerator::PrintPreamble(grpc_generator::Printer* out) { - StringMap var; - var["Package"] = config.grpc_package_root; - out->Print(var, "import $Package$\n"); - out->Print("\n"); - StringPairSet imports_set; - for (int i = 0; i < file->service_count(); ++i) { - auto service = file->service(i); - for (int j = 0; j < service->method_count(); ++j) { - auto method = service.get()->method(j); - - grpc::string input_type_file_name = method->get_fb_builder(); - grpc::string input_module_name = - ModuleName(input_type_file_name, config.import_prefix); - grpc::string input_module_alias = - ModuleAlias(input_type_file_name, config.import_prefix); - imports_set.insert( - std::make_tuple(input_module_name, input_module_alias)); - - grpc::string output_type_file_name = method->get_fb_builder(); - grpc::string output_module_name = - ModuleName(output_type_file_name, config.import_prefix); - grpc::string output_module_alias = - ModuleAlias(output_type_file_name, config.import_prefix); - imports_set.insert( - std::make_tuple(output_module_name, output_module_alias)); - } - } - - for (StringPairSet::iterator it = imports_set.begin(); - it != imports_set.end(); ++it) { - var["ModuleName"] = std::get<0>(*it); - var["ModuleAlias"] = std::get<1>(*it); - out->Print(var, "import $ModuleName$ as $ModuleAlias$\n"); - } -} - -void PrivateGenerator::PrintGAServices(grpc_generator::Printer* out) { - grpc::string package = file->package(); - if (!package.empty()) { - package = package.append("."); - } - - out->Print(file->additional_headers().c_str()); - - for (int i = 0; i < file->service_count(); ++i) { - auto service = file->service(i); - - grpc::string package_qualified_service_name = package + service->name(); - PrintStub(package_qualified_service_name, service.get(), out); - PrintServicer(service.get(), out); - PrintAddServicerToServer(package_qualified_service_name, service.get(), - out); - } -} - -void PrivateGenerator::PrintBetaServices(grpc_generator::Printer* out) { - grpc::string package = file->package(); - if (!package.empty()) { - package = package.append("."); - } - for (int i = 0; i < file->service_count(); ++i) { - auto service = file->service(i); - - grpc::string package_qualified_service_name = package + service->name(); - PrintBetaServicer(service.get(), out); - PrintBetaStub(service.get(), out); - PrintBetaServerFactory(package_qualified_service_name, service.get(), out); - PrintBetaStubFactory(package_qualified_service_name, service.get(), out); - } -} - -grpc::string PrivateGenerator::GetGrpcServices() { + printer->Outdent(); + printer->Print("}\n"); + printer->Print(vars, "generic_handler = grpc.method_handlers_generic_handler(\n"); + printer->Indent(); + printer->Print(vars, "'$PATH$$ServiceName$', rpc_method_handlers)\n"); + printer->Outdent(); + printer->Print("server.add_generic_rpc_handlers((generic_handler,))"); + printer->Outdent(); + printer->Print("\n"); +} + +grpc::string Generate(grpc_generator::File *file, + const grpc_generator::Service *service) { grpc::string output; - { - // Scope the output stream so it closes and finalizes output to the string. - auto out = file->CreatePrinter(&output); - out->Print( - "# Generated by the gRPC Python protocol compiler plugin. " - "DO NOT EDIT!\n"); - StringMap var; - var["Package"] = config.grpc_package_root; - out->Print(var, "import $Package$\n"); - PrintGAServices(out.get()); - out->Print("try:\n"); - { - IndentScope raii_dict_try_indent(out.get()); - out->Print( - "# THESE ELEMENTS WILL BE DEPRECATED.\n" - "# Please use the generated *_pb2_grpc.py files instead.\n"); - out->Print(var, "import $Package$\n"); - PrintBetaPreamble(out.get()); - PrintGAServices(out.get()); - PrintBetaServices(out.get()); - } - out->Print("except ImportError:\n"); - { - IndentScope raii_dict_except_indent(out.get()); - out->Print("pass"); - } - } + std::map vars; + vars["PATH"] = file->package(); + if (!file->package().empty()) { vars["PATH"].append("."); } + vars["ServiceName"] = service->name(); + auto printer = file->CreatePrinter(&output); + GenerateStub(service, &*printer, &vars); + GenerateServicer(service, &*printer, &vars); + GenerateRegister(service, &*printer, &vars); return output; } diff --git a/grpc/src/compiler/python_generator.h b/grpc/src/compiler/python_generator.h index d92cb025..4f8f5cc8 100644 --- a/grpc/src/compiler/python_generator.h +++ b/grpc/src/compiler/python_generator.h @@ -21,20 +21,13 @@ #include +#include "src/compiler/config.h" #include "src/compiler/schema_interface.h" namespace grpc_python_generator { -// Data pertaining to configuration of the generator with respect to anything -// that may be used internally at Google. -struct GeneratorConfiguration { - grpc::string grpc_package_root; - // TODO(https://github.com/grpc/grpc/issues/8622): Drop this. - grpc::string beta_package_root; - // TODO(https://github.com/google/protobuf/issues/888): Drop this. - grpc::string import_prefix; -}; - +grpc::string Generate(grpc_generator::File *file, + const grpc_generator::Service *service); } // namespace grpc_python_generator #endif // GRPC_INTERNAL_COMPILER_PYTHON_GENERATOR_H diff --git a/grpc/src/compiler/python_private_generator.h b/grpc/src/compiler/python_private_generator.h deleted file mode 100644 index 30ba0d7b..00000000 --- a/grpc/src/compiler/python_private_generator.h +++ /dev/null @@ -1,72 +0,0 @@ -/* - * - * Copyright 2015 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#ifndef GRPC_INTERNAL_COMPILER_PYTHON_PRIVATE_GENERATOR_H -#define GRPC_INTERNAL_COMPILER_PYTHON_PRIVATE_GENERATOR_H - -#include -#include - -#include "src/compiler/python_generator.h" -#include "src/compiler/schema_interface.h" - -namespace grpc_python_generator { - -// Tucks all generator state in an anonymous namespace away from -// PythonGrpcGenerator and the header file, mostly to encourage future changes -// to not require updates to the grpcio-tools C++ code part. Assumes that it is -// only ever used from a single thread. -struct PrivateGenerator { - const GeneratorConfiguration& config; - const grpc_generator::File* file; - - PrivateGenerator(const GeneratorConfiguration& config, - const grpc_generator::File* file); - - grpc::string GetGrpcServices(); - - private: - void PrintPreamble(grpc_generator::Printer* out); - void PrintBetaPreamble(grpc_generator::Printer* out); - void PrintGAServices(grpc_generator::Printer* out); - void PrintBetaServices(grpc_generator::Printer* out); - - void PrintAddServicerToServer( - const grpc::string& package_qualified_service_name, - const grpc_generator::Service* service, grpc_generator::Printer* out); - void PrintServicer(const grpc_generator::Service* service, - grpc_generator::Printer* out); - void PrintStub(const grpc::string& package_qualified_service_name, - const grpc_generator::Service* service, - grpc_generator::Printer* out); - - void PrintBetaServicer(const grpc_generator::Service* service, - grpc_generator::Printer* out); - void PrintBetaServerFactory( - const grpc::string& package_qualified_service_name, - const grpc_generator::Service* service, grpc_generator::Printer* out); - void PrintBetaStub(const grpc_generator::Service* service, - grpc_generator::Printer* out); - void PrintBetaStubFactory(const grpc::string& package_qualified_service_name, - const grpc_generator::Service* service, - grpc_generator::Printer* out); -}; - -} // namespace grpc_python_generator - -#endif // GRPC_INTERNAL_COMPILER_PYTHON_PRIVATE_GENERATOR_H diff --git a/src/idl_gen_grpc.cpp b/src/idl_gen_grpc.cpp index 9beb10c5..9aea745d 100644 --- a/src/idl_gen_grpc.cpp +++ b/src/idl_gen_grpc.cpp @@ -24,7 +24,6 @@ #include "src/compiler/go_generator.h" #include "src/compiler/java_generator.h" #include "src/compiler/python_generator.h" -#include "src/compiler/python_private_generator.h" #include "src/compiler/swift_generator.h" #include "src/compiler/ts_generator.h" @@ -412,6 +411,45 @@ bool GenerateJavaGRPC(const Parser &parser, const std::string &path, return JavaGRPCGenerator(parser, path, file_name).generate(); } +class PythonGRPCGenerator : public flatbuffers::BaseGenerator { + private: + CodeWriter code_; + + public: + PythonGRPCGenerator(const Parser &parser, const std::string &filename) + : BaseGenerator(parser, "", filename, "", "" /*Unused*/, "swift") {} + + bool generate() { + code_.Clear(); + code_ += + "# Generated by the gRPC Python protocol compiler plugin. " + "DO NOT EDIT!\n"; + code_ += "import grpc\n"; + + FlatBufFile file(parser_, file_name_, FlatBufFile::kLanguagePython); + + for (int i = 0; i < file.service_count(); i++) { + auto service = file.service(i); + code_ += grpc_python_generator::Generate(&file, service.get()); + } + const auto final_code = code_.ToString(); + const auto filename = GenerateFileName(); + return SaveFile(filename.c_str(), final_code, false); + } + + std::string GenerateFileName() { + std::string namespace_dir; + auto &namespaces = parser_.namespaces_.back()->components; + for (auto it = namespaces.begin(); it != namespaces.end(); ++it) { + if (it != namespaces.begin()) namespace_dir += kPathSeparator; + namespace_dir += *it; + } + std::string grpc_py_filename = namespace_dir; + if (!namespace_dir.empty()) grpc_py_filename += kPathSeparator; + return grpc_py_filename + file_name_ + "_grpc_fb.py"; + } +}; + bool GeneratePythonGRPC(const Parser &parser, const std::string & /*path*/, const std::string &file_name) { int nservices = 0; @@ -421,28 +459,7 @@ bool GeneratePythonGRPC(const Parser &parser, const std::string & /*path*/, } if (!nservices) return true; - grpc_python_generator::GeneratorConfiguration config; - config.grpc_package_root = "grpc"; - config.beta_package_root = "grpc.beta"; - config.import_prefix = ""; - - FlatBufFile fbfile(parser, file_name, FlatBufFile::kLanguagePython); - - grpc_python_generator::PrivateGenerator generator(config, &fbfile); - - std::string code = generator.GetGrpcServices(); - std::string namespace_dir; - auto &namespaces = parser.namespaces_.back()->components; - for (auto it = namespaces.begin(); it != namespaces.end(); ++it) { - if (it != namespaces.begin()) namespace_dir += kPathSeparator; - namespace_dir += *it; - } - - std::string grpc_py_filename = namespace_dir; - if (!namespace_dir.empty()) grpc_py_filename += kPathSeparator; - grpc_py_filename += file_name + "_grpc_fb.py"; - - return flatbuffers::SaveFile(grpc_py_filename.c_str(), code, false); + return PythonGRPCGenerator(parser, file_name).generate(); } class SwiftGRPCGenerator : public flatbuffers::BaseGenerator { -- cgit v1.2.3