protobuf.bzl 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587
  1. load("@bazel_skylib//lib:versions.bzl", "versions")
  2. load("@rules_cc//cc:defs.bzl", "cc_library")
  3. load("@rules_proto//proto:defs.bzl", "ProtoInfo")
  4. load("@rules_python//python:defs.bzl", "py_library", "py_test")
  5. def _GetPath(ctx, path):
  6. if ctx.label.workspace_root:
  7. return ctx.label.workspace_root + "/" + path
  8. else:
  9. return path
  10. def _IsNewExternal(ctx):
  11. # Bazel 0.4.4 and older have genfiles paths that look like:
  12. # bazel-out/local-fastbuild/genfiles/external/repo/foo
  13. # After the exec root rearrangement, they look like:
  14. # ../repo/bazel-out/local-fastbuild/genfiles/foo
  15. return ctx.label.workspace_root.startswith("../")
  16. def _GenDir(ctx):
  17. if _IsNewExternal(ctx):
  18. # We are using the fact that Bazel 0.4.4+ provides repository-relative paths
  19. # for ctx.genfiles_dir.
  20. return ctx.genfiles_dir.path + (
  21. "/" + ctx.attr.includes[0] if ctx.attr.includes and ctx.attr.includes[0] else ""
  22. )
  23. # This means that we're either in the old version OR the new version in the local repo.
  24. # Either way, appending the source path to the genfiles dir works.
  25. return ctx.var["GENDIR"] + "/" + _SourceDir(ctx)
  26. def _SourceDir(ctx):
  27. if not ctx.attr.includes:
  28. return ctx.label.workspace_root
  29. if not ctx.attr.includes[0]:
  30. return _GetPath(ctx, ctx.label.package)
  31. if not ctx.label.package:
  32. return _GetPath(ctx, ctx.attr.includes[0])
  33. return _GetPath(ctx, ctx.label.package + "/" + ctx.attr.includes[0])
  34. def _CcHdrs(srcs, use_grpc_plugin = False):
  35. ret = [s[:-len(".proto")] + ".pb.h" for s in srcs]
  36. if use_grpc_plugin:
  37. ret += [s[:-len(".proto")] + ".grpc.pb.h" for s in srcs]
  38. return ret
  39. def _CcSrcs(srcs, use_grpc_plugin = False):
  40. ret = [s[:-len(".proto")] + ".pb.cc" for s in srcs]
  41. if use_grpc_plugin:
  42. ret += [s[:-len(".proto")] + ".grpc.pb.cc" for s in srcs]
  43. return ret
  44. def _CcOuts(srcs, use_grpc_plugin = False):
  45. return _CcHdrs(srcs, use_grpc_plugin) + _CcSrcs(srcs, use_grpc_plugin)
  46. def _PyOuts(srcs, use_grpc_plugin = False):
  47. ret = [s[:-len(".proto")] + "_pb2.py" for s in srcs]
  48. if use_grpc_plugin:
  49. ret += [s[:-len(".proto")] + "_pb2_grpc.py" for s in srcs]
  50. return ret
  51. def _RelativeOutputPath(path, include, dest = ""):
  52. if include == None:
  53. return path
  54. if not path.startswith(include):
  55. fail("Include path %s isn't part of the path %s." % (include, path))
  56. if include and include[-1] != "/":
  57. include = include + "/"
  58. if dest and dest[-1] != "/":
  59. dest = dest + "/"
  60. path = path[len(include):]
  61. return dest + path
  62. def _proto_gen_impl(ctx):
  63. """General implementation for generating protos"""
  64. srcs = ctx.files.srcs
  65. deps = depset(direct=ctx.files.srcs)
  66. source_dir = _SourceDir(ctx)
  67. gen_dir = _GenDir(ctx).rstrip("/")
  68. if source_dir:
  69. has_sources = any([src.is_source for src in srcs])
  70. has_generated = any([not src.is_source for src in srcs])
  71. import_flags = []
  72. if has_sources:
  73. import_flags += ["-I" + source_dir]
  74. if has_generated:
  75. import_flags += ["-I" + gen_dir]
  76. import_flags = depset(direct=import_flags)
  77. else:
  78. import_flags = depset(direct=["-I."])
  79. for dep in ctx.attr.deps:
  80. if type(dep.proto.import_flags) == "list":
  81. import_flags = depset(transitive=[import_flags], direct=dep.proto.import_flags)
  82. else:
  83. import_flags = depset(transitive=[import_flags, dep.proto.import_flags])
  84. if type(dep.proto.deps) == "list":
  85. deps = depset(transitive=[deps], direct=dep.proto.deps)
  86. else:
  87. deps = depset(transitive=[deps, dep.proto.deps])
  88. if not ctx.attr.gen_cc and not ctx.attr.gen_py and not ctx.executable.plugin:
  89. return struct(
  90. proto = struct(
  91. srcs = srcs,
  92. import_flags = import_flags,
  93. deps = deps,
  94. ),
  95. )
  96. for src in srcs:
  97. args = []
  98. in_gen_dir = src.root.path == gen_dir
  99. if in_gen_dir:
  100. import_flags_real = []
  101. for f in import_flags.to_list():
  102. path = f.replace("-I", "")
  103. import_flags_real.append("-I$(realpath -s %s)" % path)
  104. outs = []
  105. use_grpc_plugin = (ctx.attr.plugin_language == "grpc" and ctx.attr.plugin)
  106. path_tpl = "$(realpath %s)" if in_gen_dir else "%s"
  107. if ctx.attr.gen_cc:
  108. args += [("--cpp_out=" + path_tpl) % gen_dir]
  109. outs.extend(_CcOuts([src.basename], use_grpc_plugin = use_grpc_plugin))
  110. if ctx.attr.gen_py:
  111. args += [("--python_out=" + path_tpl) % gen_dir]
  112. outs.extend(_PyOuts([src.basename], use_grpc_plugin = use_grpc_plugin))
  113. outs = [ctx.actions.declare_file(out, sibling = src) for out in outs]
  114. inputs = [src] + deps.to_list()
  115. tools = [ctx.executable.protoc]
  116. if ctx.executable.plugin:
  117. plugin = ctx.executable.plugin
  118. lang = ctx.attr.plugin_language
  119. if not lang and plugin.basename.startswith("protoc-gen-"):
  120. lang = plugin.basename[len("protoc-gen-"):]
  121. if not lang:
  122. fail("cannot infer the target language of plugin", "plugin_language")
  123. outdir = "." if in_gen_dir else gen_dir
  124. if ctx.attr.plugin_options:
  125. outdir = ",".join(ctx.attr.plugin_options) + ":" + outdir
  126. args += [("--plugin=protoc-gen-%s=" + path_tpl) % (lang, plugin.path)]
  127. args += ["--%s_out=%s" % (lang, outdir)]
  128. tools.append(plugin)
  129. if not in_gen_dir:
  130. ctx.actions.run(
  131. inputs = inputs,
  132. tools = tools,
  133. outputs = outs,
  134. arguments = args + import_flags.to_list() + [src.path],
  135. executable = ctx.executable.protoc,
  136. mnemonic = "ProtoCompile",
  137. use_default_shell_env = True,
  138. )
  139. else:
  140. for out in outs:
  141. orig_command = " ".join(
  142. ["$(realpath %s)" % ctx.executable.protoc.path] + args +
  143. import_flags_real + ["-I.", src.basename],
  144. )
  145. command = ";".join([
  146. 'CMD="%s"' % orig_command,
  147. "cd %s" % src.dirname,
  148. "${CMD}",
  149. "cd -",
  150. ])
  151. generated_out = "/".join([gen_dir, out.basename])
  152. if generated_out != out.path:
  153. command += ";mv %s %s" % (generated_out, out.path)
  154. ctx.actions.run_shell(
  155. inputs = inputs,
  156. outputs = [out],
  157. command = command,
  158. mnemonic = "ProtoCompile",
  159. tools = tools,
  160. use_default_shell_env = True,
  161. )
  162. return struct(
  163. proto = struct(
  164. srcs = srcs,
  165. import_flags = import_flags,
  166. deps = deps,
  167. ),
  168. )
  169. proto_gen = rule(
  170. attrs = {
  171. "srcs": attr.label_list(allow_files = True),
  172. "deps": attr.label_list(providers = ["proto"]),
  173. "includes": attr.string_list(),
  174. "protoc": attr.label(
  175. cfg = "exec",
  176. executable = True,
  177. allow_single_file = True,
  178. mandatory = True,
  179. ),
  180. "plugin": attr.label(
  181. cfg = "exec",
  182. allow_files = True,
  183. executable = True,
  184. ),
  185. "plugin_language": attr.string(),
  186. "plugin_options": attr.string_list(),
  187. "gen_cc": attr.bool(),
  188. "gen_py": attr.bool(),
  189. "outs": attr.output_list(),
  190. },
  191. output_to_genfiles = True,
  192. implementation = _proto_gen_impl,
  193. )
  194. """Generates codes from Protocol Buffers definitions.
  195. This rule helps you to implement Skylark macros specific to the target
  196. language. You should prefer more specific `cc_proto_library `,
  197. `py_proto_library` and others unless you are adding such wrapper macros.
  198. Args:
  199. srcs: Protocol Buffers definition files (.proto) to run the protocol compiler
  200. against.
  201. deps: a list of dependency labels; must be other proto libraries.
  202. includes: a list of include paths to .proto files.
  203. protoc: the label of the protocol compiler to generate the sources.
  204. plugin: the label of the protocol compiler plugin to be passed to the protocol
  205. compiler.
  206. plugin_language: the language of the generated sources
  207. plugin_options: a list of options to be passed to the plugin
  208. gen_cc: generates C++ sources in addition to the ones from the plugin.
  209. gen_py: generates Python sources in addition to the ones from the plugin.
  210. outs: a list of labels of the expected outputs from the protocol compiler.
  211. """
  212. def _adapt_proto_library_impl(ctx):
  213. deps = [dep[ProtoInfo] for dep in ctx.attr.deps]
  214. srcs = [src for dep in deps for src in dep.direct_sources]
  215. return struct(
  216. proto = struct(
  217. srcs = srcs,
  218. import_flags = ["-I{}".format(path) for dep in deps for path in dep.transitive_proto_path.to_list()],
  219. deps = srcs,
  220. ),
  221. )
  222. adapt_proto_library = rule(
  223. implementation = _adapt_proto_library_impl,
  224. attrs = {
  225. "deps": attr.label_list(
  226. mandatory = True,
  227. providers = [ProtoInfo],
  228. ),
  229. },
  230. doc = "Adapts `proto_library` from `@rules_proto` to be used with `{cc,py}_proto_library` from this file.",
  231. )
  232. def cc_proto_library(
  233. name,
  234. srcs = [],
  235. deps = [],
  236. cc_libs = [],
  237. include = None,
  238. protoc = Label("//:protoc"),
  239. use_grpc_plugin = False,
  240. default_runtime = Label("//:protobuf"),
  241. **kargs):
  242. """Bazel rule to create a C++ protobuf library from proto source files
  243. NOTE: the rule is only an internal workaround to generate protos. The
  244. interface may change and the rule may be removed when bazel has introduced
  245. the native rule.
  246. Args:
  247. name: the name of the cc_proto_library.
  248. srcs: the .proto files of the cc_proto_library.
  249. deps: a list of dependency labels; must be cc_proto_library.
  250. cc_libs: a list of other cc_library targets depended by the generated
  251. cc_library.
  252. include: a string indicating the include path of the .proto files.
  253. protoc: the label of the protocol compiler to generate the sources.
  254. use_grpc_plugin: a flag to indicate whether to call the grpc C++ plugin
  255. when processing the proto files.
  256. default_runtime: the implicitly default runtime which will be depended on by
  257. the generated cc_library target.
  258. **kargs: other keyword arguments that are passed to cc_library.
  259. """
  260. includes = []
  261. if include != None:
  262. includes = [include]
  263. grpc_cpp_plugin = None
  264. if use_grpc_plugin:
  265. grpc_cpp_plugin = "//external:grpc_cpp_plugin"
  266. gen_srcs = _CcSrcs(srcs, use_grpc_plugin)
  267. gen_hdrs = _CcHdrs(srcs, use_grpc_plugin)
  268. outs = gen_srcs + gen_hdrs
  269. proto_gen(
  270. name = name + "_genproto",
  271. srcs = srcs,
  272. deps = [s + "_genproto" for s in deps],
  273. includes = includes,
  274. protoc = protoc,
  275. plugin = grpc_cpp_plugin,
  276. plugin_language = "grpc",
  277. gen_cc = 1,
  278. outs = outs,
  279. visibility = ["//visibility:public"],
  280. )
  281. if default_runtime and not default_runtime in cc_libs:
  282. cc_libs = cc_libs + [default_runtime]
  283. if use_grpc_plugin:
  284. cc_libs = cc_libs + ["//external:grpc_lib"]
  285. cc_library(
  286. name = name,
  287. srcs = gen_srcs,
  288. hdrs = gen_hdrs,
  289. deps = cc_libs + deps,
  290. includes = includes,
  291. **kargs
  292. )
  293. def _internal_gen_well_known_protos_java_impl(ctx):
  294. args = ctx.actions.args()
  295. deps = [d[ProtoInfo] for d in ctx.attr.deps]
  296. srcjar = ctx.actions.declare_file("{}.srcjar".format(ctx.attr.name))
  297. if ctx.attr.javalite:
  298. java_out = "lite:%s" % srcjar.path
  299. else:
  300. java_out = srcjar
  301. args.add("--java_out", java_out)
  302. descriptors = depset(
  303. transitive = [dep.transitive_descriptor_sets for dep in deps],
  304. )
  305. args.add_joined(
  306. "--descriptor_set_in",
  307. descriptors,
  308. join_with = ctx.configuration.host_path_separator,
  309. )
  310. for dep in deps:
  311. if "." == dep.proto_source_root:
  312. args.add_all([src.path for src in dep.direct_sources])
  313. else:
  314. source_root = dep.proto_source_root
  315. offset = len(source_root) + 1 # + '/'.
  316. args.add_all([src.path[offset:] for src in dep.direct_sources])
  317. ctx.actions.run(
  318. executable = ctx.executable._protoc,
  319. inputs = descriptors,
  320. outputs = [srcjar],
  321. arguments = [args],
  322. use_default_shell_env = True,
  323. )
  324. return [
  325. DefaultInfo(
  326. files = depset([srcjar]),
  327. ),
  328. ]
  329. internal_gen_well_known_protos_java = rule(
  330. implementation = _internal_gen_well_known_protos_java_impl,
  331. attrs = {
  332. "deps": attr.label_list(
  333. mandatory = True,
  334. providers = [ProtoInfo],
  335. ),
  336. "javalite": attr.bool(
  337. default = False,
  338. ),
  339. "_protoc": attr.label(
  340. executable = True,
  341. cfg = "exec",
  342. default = "//:protoc",
  343. ),
  344. },
  345. )
  346. def _internal_gen_kt_protos(ctx):
  347. args = ctx.actions.args()
  348. deps = [d[ProtoInfo] for d in ctx.attr.deps]
  349. srcjar = ctx.actions.declare_file("{}.srcjar".format(ctx.attr.name))
  350. if ctx.attr.lite:
  351. out = "lite:%s" % srcjar.path
  352. else:
  353. out = srcjar
  354. args.add("--kotlin_out", out)
  355. descriptors = depset(
  356. transitive = [dep.transitive_descriptor_sets for dep in deps],
  357. )
  358. args.add_joined(
  359. "--descriptor_set_in",
  360. descriptors,
  361. join_with = ctx.configuration.host_path_separator,
  362. )
  363. for dep in deps:
  364. if "." == dep.proto_source_root:
  365. args.add_all([src.path for src in dep.direct_sources])
  366. else:
  367. source_root = dep.proto_source_root
  368. offset = len(source_root) + 1 # + '/'.
  369. args.add_all([src.path[offset:] for src in dep.direct_sources])
  370. ctx.actions.run(
  371. executable = ctx.executable._protoc,
  372. inputs = descriptors,
  373. outputs = [srcjar],
  374. arguments = [args],
  375. use_default_shell_env = True,
  376. )
  377. return [
  378. DefaultInfo(
  379. files = depset([srcjar]),
  380. ),
  381. ]
  382. internal_gen_kt_protos = rule(
  383. implementation = _internal_gen_kt_protos,
  384. attrs = {
  385. "deps": attr.label_list(
  386. mandatory = True,
  387. providers = [ProtoInfo],
  388. ),
  389. "lite": attr.bool(
  390. default = False,
  391. ),
  392. "_protoc": attr.label(
  393. executable = True,
  394. cfg = "exec",
  395. default = "//:protoc",
  396. ),
  397. },
  398. )
  399. def internal_copied_filegroup(name, srcs, strip_prefix, dest, **kwargs):
  400. """Macro to copy files to a different directory and then create a filegroup.
  401. This is used by the //:protobuf_python py_proto_library target to work around
  402. an issue caused by Python source files that are part of the same Python
  403. package being in separate directories.
  404. Args:
  405. srcs: The source files to copy and add to the filegroup.
  406. strip_prefix: Path to the root of the files to copy.
  407. dest: The directory to copy the source files into.
  408. **kwargs: extra arguments that will be passesd to the filegroup.
  409. """
  410. outs = [_RelativeOutputPath(s, strip_prefix, dest) for s in srcs]
  411. native.genrule(
  412. name = name + "_genrule",
  413. srcs = srcs,
  414. outs = outs,
  415. cmd = " && ".join(
  416. ["cp $(location %s) $(location %s)" %
  417. (s, _RelativeOutputPath(s, strip_prefix, dest)) for s in srcs],
  418. ),
  419. )
  420. native.filegroup(
  421. name = name,
  422. srcs = outs,
  423. **kwargs
  424. )
  425. def py_proto_library(
  426. name,
  427. srcs = [],
  428. deps = [],
  429. py_libs = [],
  430. py_extra_srcs = [],
  431. include = None,
  432. default_runtime = Label("//:protobuf_python"),
  433. protoc = Label("//:protoc"),
  434. use_grpc_plugin = False,
  435. **kargs):
  436. """Bazel rule to create a Python protobuf library from proto source files
  437. NOTE: the rule is only an internal workaround to generate protos. The
  438. interface may change and the rule may be removed when bazel has introduced
  439. the native rule.
  440. Args:
  441. name: the name of the py_proto_library.
  442. srcs: the .proto files of the py_proto_library.
  443. deps: a list of dependency labels; must be py_proto_library.
  444. py_libs: a list of other py_library targets depended by the generated
  445. py_library.
  446. py_extra_srcs: extra source files that will be added to the output
  447. py_library. This attribute is used for internal bootstrapping.
  448. include: a string indicating the include path of the .proto files.
  449. default_runtime: the implicitly default runtime which will be depended on by
  450. the generated py_library target.
  451. protoc: the label of the protocol compiler to generate the sources.
  452. use_grpc_plugin: a flag to indicate whether to call the Python C++ plugin
  453. when processing the proto files.
  454. **kargs: other keyword arguments that are passed to py_library.
  455. """
  456. outs = _PyOuts(srcs, use_grpc_plugin)
  457. includes = []
  458. if include != None:
  459. includes = [include]
  460. grpc_python_plugin = None
  461. if use_grpc_plugin:
  462. grpc_python_plugin = "//external:grpc_python_plugin"
  463. # Note: Generated grpc code depends on Python grpc module. This dependency
  464. # is not explicitly listed in py_libs. Instead, host system is assumed to
  465. # have grpc installed.
  466. proto_gen(
  467. name = name + "_genproto",
  468. srcs = srcs,
  469. deps = [s + "_genproto" for s in deps],
  470. includes = includes,
  471. protoc = protoc,
  472. gen_py = 1,
  473. outs = outs,
  474. visibility = ["//visibility:public"],
  475. plugin = grpc_python_plugin,
  476. plugin_language = "grpc",
  477. )
  478. if default_runtime and not default_runtime in py_libs + deps:
  479. py_libs = py_libs + [default_runtime]
  480. py_library(
  481. name = name,
  482. srcs = outs + py_extra_srcs,
  483. deps = py_libs + deps,
  484. imports = includes,
  485. **kargs
  486. )
  487. def internal_protobuf_py_tests(
  488. name,
  489. modules = [],
  490. **kargs):
  491. """Bazel rules to create batch tests for protobuf internal.
  492. Args:
  493. name: the name of the rule.
  494. modules: a list of modules for tests. The macro will create a py_test for
  495. each of the parameter with the source "google/protobuf/%s.py"
  496. kargs: extra parameters that will be passed into the py_test.
  497. """
  498. for m in modules:
  499. s = "python/google/protobuf/internal/%s.py" % m
  500. py_test(
  501. name = "py_%s" % m,
  502. srcs = [s],
  503. main = s,
  504. **kargs
  505. )
  506. def check_protobuf_required_bazel_version():
  507. """For WORKSPACE files, to check the installed version of bazel.
  508. This ensures bazel supports our approach to proto_library() depending on a
  509. copied filegroup. (Fixed in bazel 0.5.4)
  510. """
  511. versions.check(minimum_bazel_version = "0.5.4")