"""Provide bazel rules for OSS CLIF."""
# Label for our OSS CLIF binary pyclif.
CLIF_PYCLIF = "@clif//:pyclif"
# Label for our OSS CLIF protobuf compiler.
CLIF_PROTO = "@clif//:proto"
# Label for our OSS CLIF C++ runtime headers and sources.
CLIF_CPP_RUNTIME = "@clif//:cpp_runtime"
# The CLIF generated code only compiles with C++17.
EXTRA_CC_FLAGS = ["-std=c++17"]
_PROTO_LIBRARY_SUFFIX = "_pyclif"
PYCLIF_PYEXT_SUFFIX = ".so"
PYCLIF_CC_LIB_SUFFIX = "_cclib"
PYCLIF_WRAP_SUFFIX = "_clif_wrap"
def _get_transitive_headers(hdrs, deps):
"""Obtain the header files for a target and its transitive dependencies.
Args:
hdrs: a list of header files
deps: a list of targets that are direct dependencies
Returns:
a collection of the transitive headers
"""
return depset(
hdrs,
transitive = [dep[CcInfo].compilation_context.headers for dep in deps],
)
def _clif_wrap_cc_impl(ctx):
"""Executes CLIF cmdline tool to produce C++ python model from a CLIF spec."""
if len(ctx.files.srcs) != 1:
fail("Exactly one CLIF source file label must be specified.", "srcs")
clif_spec_file = ctx.files.srcs[0]
# Inputs is a set of all of the things we depend on, not inputs to the CLIF
# program itself.
initial_inputs = [clif_spec_file]
initial_inputs += ctx.files._cliflib
initial_inputs += ctx.files.clif_deps
initial_inputs += ctx.files.toolchain_deps
inputs = _get_transitive_headers(initial_inputs, ctx.attr.deps)
# Compute the set of include directories for CLIF so it can find header files
# used in the CLIF specification. These are the repo roots for all of our
# inputs (aka deps) plus all of the quote and system includes for our C++
# deps.
include_dirs = depset(
_get_repository_roots(ctx, inputs),
transitive = [
dep[CcInfo].compilation_context.quote_includes
for dep in ctx.attr.deps
] + [
dep[CcInfo].compilation_context.system_includes
for dep in ctx.attr.deps
],
)
# Construct our arguments for CLIF.
args = [
"--modname",
ctx.attr.package_name + "." + ctx.attr.module_name,
"-c",
ctx.outputs.cc_out.path,
"-g",
ctx.outputs.h_out.path,
"-i",
ctx.outputs.ccinit_out.path,
"--prepend",
"clif/python/types.h",
]
include_args = ["-I" + i for i in include_dirs.to_list()]
# Add these includes to CLIF itself.
args += include_args
# Add these includes to those passed through by CLIF to its C++ matcher.
args += ["-f" + " ".join(include_args + EXTRA_CC_FLAGS)]
# The last argument is the actual CLIF specification file.
args += [clif_spec_file.path]
outputs = [ctx.outputs.cc_out, ctx.outputs.h_out, ctx.outputs.ccinit_out]
ctx.actions.run(
executable = ctx.executable._clif,
arguments = args,
inputs = inputs.to_list(),
outputs = outputs,
mnemonic = "CLIF",
progress_message = "CLIF wrapping " + clif_spec_file.path,
)
_clif_wrap_cc = rule(
attrs = {
"srcs": attr.label_list(
mandatory = True,
allow_files = True,
),
"deps": attr.label_list(
allow_files = True,
providers = [CcInfo],
),
"toolchain_deps": attr.label_list(
allow_files = True,
),
# For rule "//foo/python:bar_clif" this should be "bar".
"module_name": attr.string(mandatory = True),
# For rule "//foo/python:bar_clif" this should be "foo/python".
"package_name": attr.string(mandatory = True),
"clif_deps": attr.label_list(allow_files = True),
# Hidden attribute: the Label for our PYCLIF binary itself.
"_clif": attr.label(
default = Label(CLIF_PYCLIF),
executable = True,
cfg = "exec",
),
# Hidden attribute: The label to the C++ CLIF header files.
"_cliflib": attr.label(
default = Label(CLIF_CPP_RUNTIME),
allow_files = True,
),
},
output_to_genfiles = True,
outputs = {
"cc_out": "%{module_name}.cc",
"h_out": "%{module_name}.h",
"ccinit_out": "%{module_name}_init.cc",
},
implementation = _clif_wrap_cc_impl,
)
def _get_repository_roots(ctx, files):
"""Returns abnormal root directories under which files reside.
When running a ctx.action, source files within the main repository are all
relative to the current directory; however, files that are generated or exist
in remote repositories will have their root directory be a subdirectory,
e.g. bazel-out/local-fastbuild/genfiles/external/jpeg_archive. This function
returns the set of these devious directories, ranked and sorted by popularity
in order to hopefully minimize the number of I/O system calls within the
compiler, because includes have quadratic complexity.
Args:
ctx: context
files: list of paths
Returns:
list of directories
"""
ctx = ctx # unused
result = {}
for f in files.to_list():
root = f.root.path
if root:
if root not in result:
result[root] = 0
result[root] -= 1
work = f.owner.workspace_root
if work:
if root:
root += "/"
root += work
if root:
if root not in result:
result[root] = 0
result[root] -= 1
return [k for v, k in sorted([(v, k) for k, v in result.items()])]
def _clif_to_lib(label, extension):
"""Gets a C++/python/etc library corresponding to a CLIF library rule.
Args:
label: string. The name of a clif_rule. If the name is of the
form <target>_pyclif we will stripe off the `_pyclif` ending.
extension: string. The expected extension of our name library.
Returns:
<target>_extension.
"""
if label.endswith(_PROTO_LIBRARY_SUFFIX):
basename = label[:-len(_PROTO_LIBRARY_SUFFIX)]
else:
basename = label
return basename + extension
def pyclifs_to_pyproto_libs(labels):
"""Gets the py protobuf label for each of pyclif label as a list."""
return [_clif_to_lib(name, "_py_pb2") for name in labels]
def pyclifs_to_ccproto_libs(labels):
"""Gets the cc protobuf label for each of pyclif label as a list."""
return [_clif_to_lib(name, "_cc_pb2") for name in labels]
def clif_deps_to_cclibs(labels):
"""Gets the cc_library name for each of label as a list."""
return [_clif_to_lib(name, PYCLIF_CC_LIB_SUFFIX) for name in labels]
def _symlink_impl(ctx):
"""Creates a symbolic link between src and out."""
out = ctx.outputs.out
src = ctx.attr.src.files.to_list()[0]
cmd = "ln -f -r -s %s %s" % (src.path, out.path)
ctx.actions.run_shell(
inputs = [src],
outputs = [out],
command = cmd,
)
symlink = rule(
implementation = _symlink_impl,
attrs = {
"src": attr.label(
mandatory = True,
allow_single_file = True,
),
"out": attr.output(mandatory = True),
},
)
def py_clif_cc(
name,
srcs,
clif_deps = [],
pyclif_deps = [],
deps = [],
copts = [],
py_deps = [],
**kwargs):
"""Defines a CLIF wrapper rule making C++ libraries accessible to Python.
Here are two example working py_clif_cc rules:
py_clif_cc(
name = "proto_cpp",
srcs = ["proto_cpp.clif"],
pyclif_deps = ["//oss_clif:oss_pyclif"],
deps = ["//oss_clif:proto_cpp_lib"],
)
py_clif_cc(
name = "pyclif_dep",
srcs = ["pyclif_dep.clif"],
deps = ["//oss_clif:pyclif_dep_lib"],
)
Args:
name: The name of the rule. This name becomes a suitable target for Python
libraries to access the C++ code.
srcs: A list that must contain a single file named <name>.clif containing
our CLIF specification.
clif_deps: A list of other CLIF rules included by this one.
pyclif_deps: A potentially empty list of pyclif_proto_library rules
deps: A list of C++ dependencies.
copts: List of copts to provide to our native.cc_library when building our
python extension module.
py_deps: List of dependencies to provide to our the native.py_library
created by this rule.
**kwargs: kwargs passed to py_library rule created by this rule.
"""
pyext_so = name + PYCLIF_PYEXT_SUFFIX
cc_library_name = name + PYCLIF_CC_LIB_SUFFIX
extended_cc_deps = deps + [CLIF_CPP_RUNTIME] + pyclif_deps
# Here's a rough outline of how we build our pyclif library:
#
# Suppose we have a module named 'foo'.
#
# _clif_wrap_cc runs pyclif to produce foo.cc, foo.h, and foo_init.cc which
# C++ python extension module.
#
# native.cc_library is a normal C++ library with those sources, effectively
# our "python module" as a bazel C++ library allowing other rules to depend
# on that C++ code.
#
# native.cc_binary depends on foo's cc_library to create a shared python
# extension module (.so) which python will load via its dlopen mechanism.
# This .so library is also used by the _clif_wrap_cc rule to get include paths
# when building CLIF specs depending on other clif specs.
#
# native.py_library named `name` which provides a python bazel target that
# loads the cc_binary, as data, producing a py extension module. This also
# allows client python code to depend on this module.
# This _clif_wrap_cc handles clif_deps differently than most clif.bzl's
# do. That is because we are doing something quite different than the
# standard clif.bzl does -- we are replacing the generated extension module
# with a symbolic link to protobuf's _message.so. That file, in turn, is
# a dependency of every single Python protocol buffer target. And it needs
# to depend on all of the _cclibs generated by the cc_library rule below.
#
# So because of all this, the normal strategy of having this rule depend
# on clif_deps or clif_deps_to_pyexts(clif_deps) would create circular
# dependencies. Our strategy to avoid those circular dependencies is to
# have anything the cc_library rule depends on be only C++ and not Python.
# The raw clif_deps are a mixture of C++ and Python, so we have to very
# carefully depend on only their C++ part (given by the
# clif_deps_to_cclibs).
_clif_wrap_cc(
name = name + PYCLIF_WRAP_SUFFIX,
srcs = srcs,
deps = extended_cc_deps + clif_deps_to_cclibs(clif_deps),
clif_deps = clif_deps_to_cclibs(clif_deps),
toolchain_deps = ["@bazel_tools//tools/cpp:current_cc_toolchain"],
module_name = name,
# Turns //foo/bar:baz_pyclif into foo.bar to create our fully-qualified
# python package name.
package_name = native.package_name().replace("/", "."),
)
native.cc_library(
name = cc_library_name,
hdrs = [
name + ".h",
],
srcs = [
name + ".cc",
name + "_init.cc",
],
copts = copts + EXTRA_CC_FLAGS,
deps = extended_cc_deps + clif_deps_to_cclibs(clif_deps),
)
# To prevent ODR violations, all of the extensions must live in one
# extension module. And to be compatible with existing protobuf
# generated code, that module must be _message.so.
symlink(
name = name + "_symlink",
out = pyext_so,
src = "@com_google_protobuf//:python/google/protobuf/pyext/_message.so",
)
# We create our python module which is just a thin wrapper around our real
# python module pyext_so (producing name.so for python to load). This
# rule allows python code to depend on this module, even through its written
# in C++.
native.py_library(
name = name,
srcs = [],
srcs_version = "PY3",
deps = pyclifs_to_pyproto_libs(pyclif_deps) + clif_deps + py_deps,
data = [pyext_so],
**kwargs
)
# Copied from: devtools/clif/python/clif_build_rule.bzl with heavy
# modifications.
def _clif_proto_parser_rule_impl(ctx):
"""Implementation of _run_clif_proto_parser_rule."""
proto_file = ctx.files.src[0]
args = [
"-c",
ctx.outputs.cc.path,
"-h",
ctx.outputs.hdr.path,
"--strip_dir=%s" % ctx.configuration.genfiles_dir.path,
"--source_dir='.'",
"%s" % proto_file.path,
]
inputs = []
for d in ctx.attr.deps:
if "proto" in dir(d):
inputs += list(d[ProtoInfo].transitive_sources)
ctx.actions.run(
mnemonic = "ClifProtoLibraryGeneration",
arguments = args,
executable = ctx.executable.parser,
inputs = inputs,
outputs = [ctx.outputs.hdr, ctx.outputs.cc],
)
_run_clif_proto_parser_rule = rule(
attrs = {
"src": attr.label(allow_files = [".proto"]),
"hdr": attr.output(),
"cc": attr.output(),
"deps": attr.label_list(),
"parser": attr.label(
executable = True,
default = Label(CLIF_PROTO),
cfg = "exec",
),
},
output_to_genfiles = True,
implementation = _clif_proto_parser_rule_impl,
)
def pyclif_proto_library(
name,
proto_lib,
proto_srcfile = "",
deps = [],
visibility = None,
compatible_with = None,
testonly = None):
"""Generate C++ CLIF extension for using a proto and dependent py_proto_lib.
Args:
name: generated cc_library (name.h) to use in py_clif_cc clif_deps
proto_lib: name of a proto_library rule
proto_srcfile: the proto name if it does not match proto_lib rule name
deps: passed to cc_library
visibility: passed to all generated "files": name.h name.a name_pb2.py
compatible_with: compatibility list
testonly: available for test rules only flag (default from package)
"""
if not name.endswith(_PROTO_LIBRARY_SUFFIX):
fail("The name of the 'pyclif_proto_library' target should be of the " +
"form '<PROTO_FILE>%s' where the proto " % _PROTO_LIBRARY_SUFFIX +
"file being wrapped has the name '<PROTO_FILE>.proto'.")
if proto_srcfile:
required_name = proto_srcfile[:-len(".proto")] + _PROTO_LIBRARY_SUFFIX
if name != required_name:
fail("The name of the 'pyclif_proto_library' target should be " +
"'%s' as it is wrapping %s." % (required_name, proto_srcfile))
hdr_file = name + ".h"
cpp_file = name + ".cc"
clifrule = name + "_clif_rule"
src = name[:-len(_PROTO_LIBRARY_SUFFIX)] + ".proto"
_run_clif_proto_parser_rule(
name = clifrule,
src = src,
hdr = hdr_file,
cc = cpp_file,
deps = deps + [proto_lib],
testonly = testonly,
)
# In OSS world, we cannot provide proto_lib as a direct dependency to our
# cc_library as it doesn't provide a cc file:
# in deps attribute of cc_library rule //oss_clif:oss_pyclif: proto_library
# rule '//oss_clif:oss_proto' is misplaced here (expected cc_inc_library,
# cc_library, objc_library, experimental_objc_library or cc_proto_library).
# So we need to synthesize our protobuf cc library name from our name as
# pyclif_name = proto_pyclif
# cc_proto_lib = proto_cc_pb2
native.cc_library(
name = name,
srcs = [cpp_file],
hdrs = [hdr_file],
deps = deps + [CLIF_CPP_RUNTIME] + pyclifs_to_ccproto_libs([name]),
visibility = visibility,
compatible_with = compatible_with,
testonly = testonly,
copts = EXTRA_CC_FLAGS,
)
