Diff of /build_release_binaries.sh [000000] .. [5a4941]

Switch to side-by-side view

--- a
+++ b/build_release_binaries.sh
@@ -0,0 +1,181 @@
+#!/bin/bash
+# Copyright 2017 Google LLC.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+#    this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its
+#    contributors may be used to endorse or promote products derived from this
+#    software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+#
+# Build release binaries using our standard compiler flags.
+# Note we use the lowest common denominator compiler optimization options (For
+# Google Cloud Engine chipsets lowest possible is Sandy Bridge).
+
+# NOLINT
+source settings.sh
+
+set -e
+
+# Bazel's --build_python_zip replaces our carefully engineered symbolic links
+# with copies.  This function puts the symbolic links back.
+function fix_zip_file {
+  orig_zip_file=$1
+
+  # Step 1:  Copy the zip file to a temporary place.
+  TMPDIR=$(mktemp -d -t tmp.XXXXXXXXXXX)
+  # The .zip version of the binary doesn't have the header that makes it
+  # self-executable.  We use that version because otherwise unzip would
+  # complain and raise an error code.
+  cp "${orig_zip_file}.zip" "${TMPDIR}"
+
+  # Step 2: Unzip it.
+  pushd "${TMPDIR}" > /dev/null
+  BN=$(basename "${orig_zip_file}")
+  unzip -qq "${BN}.zip"
+
+  # Step 3: Restore the symbolic links.
+  find "runfiles/com_google_deepvariant" -name '*.so' ! -name 'examples_from_stream.so' -exec ln --force -s --relative "runfiles/com_google_protobuf/python/google/protobuf/pyext/_message.so" {} \;
+
+  # Step 4: Fix the __main__.py's use of zipfile, which can't handle
+  # symbolic links.  Replace it with an invocation of unzip, which can.
+  # The lines we replace are
+  # with zipfile.ZipFile(zip_path) as zf:
+  #   for info in zf.infolist():
+  #     zf.extract(info, dest_dir)
+  #     # UNC-prefixed paths must be absolute/normalized. See
+
+  sed -i 's/  with zipfile.ZipFile(zip_path) as zf:/  if True:/' __main__.py
+  sed -i 's/  for info in zf.infolist():/  if True:/' __main__.py
+  sed -i 's/  zf.extract(info, dest_dir)/  os.system("unzip -qq " + zip_path + " -d " + dest_dir)/' __main__.py
+  sed -i 's/  # UNC-prefixed paths must be absolute\/normalized. See/  return/' __main__.py
+
+  # Step 5: Zip it back up, with zip --symbolic
+  rm -f "${BN}.zip"
+  ZIP_OUT="/tmp/${BN}.zip"
+  rm -f "${ZIP_OUT}"
+  zip -q --symlinks -r "${ZIP_OUT}" *
+
+  # Step 6: Make the zip file self-executable
+  SELF_ZIP="/tmp/${BN}"
+  # If the Python interpreter discovers it is being run from part of a zip
+  # file, it will uncompress and run the __main__.py.  This is the trick that
+  # bazel uses to make a self-executable zip, see for example
+  # https://github.com/bazelbuild/bazel/blob/558b717e906156477b1c6bd29d049a0fb8e18b27/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java#L193
+  echo '#!/usr/bin/env python3' | cat - "${ZIP_OUT}" > "${SELF_ZIP}"
+
+  # Step 7: Copy it back and make it executable.
+  popd > /dev/null
+  rm -f "${orig_zip_file}"
+  mv "${SELF_ZIP}" "${orig_zip_file}"
+  chmod +x "${orig_zip_file}"
+
+  # Step 8: DeepVariant also uses "${orig_zip_file}.zip" in many of its
+  # instructions, so make sure that we also copy that.
+  rm -f "${orig_zip_file}.zip"
+  mv "${ZIP_OUT}" "${orig_zip_file}.zip"
+  # No executable bit because the .zip version is not self-executing and
+  # must be invoked as
+  #   python3 ${orig_zip_file}.zip
+}
+
+# Building examples_from_stream.so C++ library. It cannot be built correctly
+# with the default bazel setup, so we build it manually.
+# examples_from_stream.so is used by call_variants target therefore it has to
+# be built before :binaries.
+TF_CFLAGS=( $(python3 -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_compile_flags()))') )
+TF_LFLAGS=( $(python3 -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_link_flags()))') )
+
+# shellcheck disable=SC2068
+g++ -std=c++14 -shared \
+        deepvariant/stream_examples_kernel.cc  \
+        deepvariant/stream_examples_ops.cc \
+        -o deepvariant/examples_from_stream.so \
+        -fPIC \
+        -l:libtensorflow_framework.so.2  \
+        -I. \
+        ${TF_CFLAGS[@]} \
+        ${TF_LFLAGS[@]} \
+        -D_GLIBCXX_USE_CXX11_ABI=1 \
+        --std=c++17 \
+        -DEIGEN_MAX_ALIGN_BYTES=64 \
+        -O2
+
+# shellcheck disable=SC2086
+bazel build -c opt \
+  //deepvariant:fast_pipeline
+
+# shellcheck disable=SC2086
+bazel build -c opt \
+  --output_filter=DONT_MATCH_ANYTHING \
+  --noshow_loading_progress \
+  --show_result=0 \
+  ${DV_COPT_FLAGS} \
+  --build_python_zip \
+  :binaries
+
+# shellcheck disable=SC2086
+bazel build -c opt \
+  --output_filter=DONT_MATCH_ANYTHING \
+  --noshow_loading_progress \
+  --show_result=0 \
+  ${DV_COPT_FLAGS} \
+  --build_python_zip \
+  //deepvariant/labeler:labeled_examples_to_vcf
+
+# shellcheck disable=SC2086
+bazel build -c opt \
+  --output_filter=DONT_MATCH_ANYTHING \
+  --noshow_loading_progress \
+  --show_result=0 \
+  ${DV_COPT_FLAGS} \
+  --build_python_zip \
+  :binaries-deeptrio
+
+# shellcheck disable=SC2086
+bazel build  -c opt \
+  --output_filter=DONT_MATCH_ANYTHING \
+  --noshow_loading_progress \
+  --show_result=0 \
+  --noshow_progress \
+  ${DV_COPT_FLAGS} \
+  :licenses_zip
+
+# Bazel understandably doesn't like it when its output files are edited, so
+# make sure all the builds are done before we fix things.
+
+# TODO: Replace this hand-made list with a find command.
+fix_zip_file "bazel-out/k8-opt/bin/deepvariant/train"
+fix_zip_file "bazel-out/k8-opt/bin/deepvariant/call_variants"
+fix_zip_file "bazel-out/k8-opt/bin/deepvariant/load_gbz_into_shared_memory"
+fix_zip_file "bazel-out/k8-opt/bin/deepvariant/make_examples"
+fix_zip_file "bazel-out/k8-opt/bin/deepvariant/make_examples_pangenome_aware_dv"
+fix_zip_file "bazel-out/k8-opt/bin/deepvariant/make_examples_somatic"
+fix_zip_file "bazel-out/k8-opt/bin/deeptrio/make_examples"
+fix_zip_file "bazel-out/k8-opt/bin/deepvariant/postprocess_variants"
+fix_zip_file "bazel-out/k8-opt/bin/deepvariant/vcf_stats_report"
+fix_zip_file "bazel-out/k8-opt/bin/deepvariant/show_examples"
+fix_zip_file "bazel-out/k8-opt/bin/deepvariant/runtime_by_region_vis"
+fix_zip_file "bazel-out/k8-opt/bin/deepvariant/multisample_make_examples"
+fix_zip_file "bazel-out/k8-opt/bin/deepvariant/labeler/labeled_examples_to_vcf"