Compatibility with tensorflow master (#204)

* TF defines the same external repositories but with different commit and BUILD

* Fix a bunch of bazel namespace issues

@tf changed to @org_tensorflow
Tensorflow stopped using protobuf as submodule, now it's external, so use @protobuf

* Sync to master of tensorflow submodule:
  - local version of is_cuda in syntaxnet.bzl,
  - using unique_ptr in file api.

Author: dmansfield

syntaxnet/WORKSPACE

@@ -1,36 +1,36 @@
 local_repository(
-  name = "tf",
+  name = "org_tensorflow",
   path = __workspace_dir__ + "/tensorflow",
 )
 
 load('//tensorflow/tensorflow:workspace.bzl', 'tf_workspace')
-tf_workspace("tensorflow/", "@tf")
+tf_workspace("tensorflow/", "@org_tensorflow")
 
 # Specify the minimum required Bazel version.
-load("@tf//tensorflow:tensorflow.bzl", "check_version")
+load("@org_tensorflow//tensorflow:tensorflow.bzl", "check_version")
 check_version("0.2.0")
 
 # ===== gRPC dependencies =====
 
 bind(
     name = "libssl",
-    actual = "@boringssl_git//:ssl",
+    actual = "@ts_boringssl_git//:ssl",
 )
 
 git_repository(
-    name = "boringssl_git",
+    name = "ts_boringssl_git",
     commit = "436432d849b83ab90f18773e4ae1c7a8f148f48d",
     init_submodules = True,
     remote = "https://github.com/mdsteele/boringssl-bazel.git",
 )
 
 bind(
     name = "zlib",
-    actual = "@zlib_archive//:zlib",
+    actual = "@ts_zlib_archive//:zlib",
 )
 
 new_http_archive(
-    name = "zlib_archive",
+    name = "ts_zlib_archive",
     build_file = "zlib.BUILD",
     sha256 = "879d73d8cd4d155f31c1f04838ecd567d34bebda780156f0e82a20721b3973d5",
     strip_prefix = "zlib-1.2.8",

syntaxnet/syntaxnet/BUILD

@@ -77,15 +77,15 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         "@re2//:re2",
-        "@tf//google/protobuf",
-        "@tf//third_party/eigen3",
+        "@protobuf//:protobuf",
+        "@org_tensorflow//third_party/eigen3",
     ] + select({
         "//conditions:default": [
-            "@tf//tensorflow/core:framework",
-            "@tf//tensorflow/core:lib",
+            "@org_tensorflow//tensorflow/core:framework",
+            "@org_tensorflow//tensorflow/core:lib",
         ],
-        "@tf//tensorflow:darwin": [
-            "@tf//tensorflow/core:framework_headers_lib",
+        "@org_tensorflow//tensorflow:darwin": [
+            "@org_tensorflow//tensorflow/core:framework_headers_lib",
         ],
     }),
 )
@@ -108,8 +108,8 @@ cc_library(
     srcs = ["test_main.cc"],
     linkopts = ["-lm"],
     deps = [
-        "@tf//tensorflow/core:lib",
-        "@tf//tensorflow/core:testlib",
+        "@org_tensorflow//tensorflow/core:lib",
+        "@org_tensorflow//tensorflow/core:testlib",
         "//external:gtest",
     ],
 )
@@ -409,7 +409,7 @@ cc_binary(
     name = "parser_ops.so",
     linkopts = select({
         "//conditions:default": ["-lm"],
-        "@tf//tensorflow:darwin": [],
+        "@org_tensorflow//tensorflow:darwin": [],
     }),
     linkshared = 1,
     linkstatic = 1,
@@ -519,8 +519,8 @@ py_library(
     name = "graph_builder",
     srcs = ["graph_builder.py"],
     deps = [
-        "@tf//tensorflow:tensorflow_py",
-        "@tf//tensorflow/core:protos_all_py",
+        "@org_tensorflow//tensorflow:tensorflow_py",
+        "@org_tensorflow//tensorflow/core:protos_all_py",
         ":load_parser_ops_py",
         ":parser_ops",
     ],

syntaxnet/syntaxnet/proto_io.h

@@ -43,18 +43,19 @@ namespace syntaxnet {
 // A convenience wrapper to read protos with a RecordReader.
 class ProtoRecordReader {
  public:
-  explicit ProtoRecordReader(tensorflow::RandomAccessFile *file)
-      : file_(file), reader_(new tensorflow::io::RecordReader(file_)) {}
+  explicit ProtoRecordReader(tensorflow::RandomAccessFile *file) {
+    file_.reset(file);
+    reader_.reset(new tensorflow::io::RecordReader(file_.get()));
+  }
 
   explicit ProtoRecordReader(const string &filename) {
     TF_CHECK_OK(
         tensorflow::Env::Default()->NewRandomAccessFile(filename, &file_));
-    reader_.reset(new tensorflow::io::RecordReader(file_));
+    reader_.reset(new tensorflow::io::RecordReader(file_.get()));
   }
 
   ~ProtoRecordReader() {
     reader_.reset();
-    delete file_;
   }
 
   template <typename T>
@@ -70,22 +71,22 @@ class ProtoRecordReader {
   }
 
  private:
-  tensorflow::RandomAccessFile *file_ = nullptr;
   uint64 offset_ = 0;
   std::unique_ptr<tensorflow::io::RecordReader> reader_;
+  std::unique_ptr<tensorflow::RandomAccessFile> file_;
 };
 
 // A convenience wrapper to write protos with a RecordReader.
 class ProtoRecordWriter {
  public:
   explicit ProtoRecordWriter(const string &filename) {
     TF_CHECK_OK(tensorflow::Env::Default()->NewWritableFile(filename, &file_));
-    writer_.reset(new tensorflow::io::RecordWriter(file_));
+    writer_.reset(new tensorflow::io::RecordWriter(file_.get()));
   }
 
   ~ProtoRecordWriter() {
     writer_.reset();
-    delete file_;
+    file_.reset();
   }
 
   template <typename T>
@@ -94,8 +95,8 @@ class ProtoRecordWriter {
   }
 
  private:
-  tensorflow::WritableFile *file_ = nullptr;
   std::unique_ptr<tensorflow::io::RecordWriter> writer_;
+  std::unique_ptr<tensorflow::WritableFile> file_;
 };
 
 // A file implementation to read from stdin.
@@ -176,22 +177,24 @@ class TextReader {
 
   void Reset() {
     sentence_count_ = 0;
-    tensorflow::RandomAccessFile *file;
     if (filename_ == "-") {
       static const int kInputBufferSize = 8 * 1024; /* bytes */
-      file = new StdIn();
-      buffer_.reset(new tensorflow::io::InputBuffer(file, kInputBufferSize));
+      file_.reset(new StdIn());
+      buffer_.reset(
+          new tensorflow::io::InputBuffer(file_.get(), kInputBufferSize));
     } else {
       static const int kInputBufferSize = 1 * 1024 * 1024; /* bytes */
       TF_CHECK_OK(
-          tensorflow::Env::Default()->NewRandomAccessFile(filename_, &file));
-      buffer_.reset(new tensorflow::io::InputBuffer(file, kInputBufferSize));
+          tensorflow::Env::Default()->NewRandomAccessFile(filename_, &file_));
+      buffer_.reset(
+          new tensorflow::io::InputBuffer(file_.get(), kInputBufferSize));
     }
   }
 
  private:
   string filename_;
   int sentence_count_ = 0;
+  std::unique_ptr<tensorflow::RandomAccessFile> file_;
   std::unique_ptr<tensorflow::io::InputBuffer> buffer_;
   std::unique_ptr<DocumentFormat> format_;
 };
@@ -217,7 +220,6 @@ class TextWriter {
   ~TextWriter() {
     if (file_) {
       file_->Close();
-      delete file_;
     }
   }
 
@@ -234,7 +236,7 @@ class TextWriter {
  private:
   string filename_;
   std::unique_ptr<DocumentFormat> format_;
-  tensorflow::WritableFile *file_ = nullptr;
+  std::unique_ptr<tensorflow::WritableFile> file_;
 };
 
 }  // namespace syntaxnet

syntaxnet/syntaxnet/reader_ops.cc

@@ -514,19 +514,16 @@ class WordEmbeddingInitializer : public OpKernel {
   static tensorflow::Status CopyToTmpPath(const string &source_path,
                                           string *tmp_path) {
     // Opens source file.
-    tensorflow::RandomAccessFile *source_file;
+    std::unique_ptr<tensorflow::RandomAccessFile> source_file;
     TF_RETURN_IF_ERROR(tensorflow::Env::Default()->NewRandomAccessFile(
         source_path, &source_file));
-    std::unique_ptr<tensorflow::RandomAccessFile> source_file_deleter(
-        source_file);
 
     // Creates destination file.
-    tensorflow::WritableFile *target_file;
+    std::unique_ptr<tensorflow::WritableFile> target_file;
     *tmp_path = tensorflow::strings::Printf(
         "/tmp/%d.%lld", getpid(), tensorflow::Env::Default()->NowMicros());
     TF_RETURN_IF_ERROR(
         tensorflow::Env::Default()->NewWritableFile(*tmp_path, &target_file));
-    std::unique_ptr<tensorflow::WritableFile> target_file_deleter(target_file);
 
     // Performs copy.
     tensorflow::Status s;

syntaxnet/syntaxnet/sentence_features.cc

@@ -120,9 +120,9 @@ string AffixTableFeature::WorkspaceName() const {
 static AffixTable *CreateAffixTable(const string &filename,
                                     AffixTable::Type type) {
   AffixTable *affix_table = new AffixTable(type, 1);
-  tensorflow::RandomAccessFile *file;
+  std::unique_ptr<tensorflow::RandomAccessFile> file;
   TF_CHECK_OK(tensorflow::Env::Default()->NewRandomAccessFile(filename, &file));
-  ProtoRecordReader reader(file);
+  ProtoRecordReader reader(file.release());
   affix_table->Read(&reader);
   return affix_table;
 }

syntaxnet/syntaxnet/syntaxnet.bzl

@@ -13,26 +13,21 @@
 # limitations under the License.
 # ==============================================================================
 
-load("@tf//google/protobuf:protobuf.bzl", "cc_proto_library")
-load("@tf//google/protobuf:protobuf.bzl", "py_proto_library")
+load("@protobuf//:protobuf.bzl", "cc_proto_library")
+load("@protobuf//:protobuf.bzl", "py_proto_library")
 
 def if_cuda(if_true, if_false = []):
-    """Shorthand for select()'ing on whether we're building with CUDA.
-
-    Returns a select statement which evaluates to if_true if we're building
-    with CUDA enabled.  Otherwise, the select statement evaluates to if_false.
-
-    """
+    """Shorthand for select()'ing on whether we're building with CUDA."""
     return select({
-        "@tf//third_party/gpus/cuda:using_nvcc": if_true,
-        "@tf//third_party/gpus/cuda:using_gcudacc": if_true,
+        "@org_tensorflow//third_party/gpus/cuda:using_nvcc": if_true,
+        "@org_tensorflow//third_party/gpus/cuda:using_gcudacc": if_true,
         "//conditions:default": if_false
     })
 
 def tf_copts():
   return (["-fno-exceptions", "-DEIGEN_AVOID_STL_ARRAY",] +
           if_cuda(["-DGOOGLE_CUDA=1"]) +
-          select({"@tf//tensorflow:darwin": [],
+          select({"@org_tensorflow//tensorflow:darwin": [],
                   "//conditions:default": ["-pthread"]}))
 
 def tf_proto_library(name, srcs=[], has_services=False,
@@ -47,9 +42,9 @@ def tf_proto_library(name, srcs=[], has_services=False,
   cc_proto_library(name=name,
                    srcs=srcs,
                    deps=deps,
-                   cc_libs = ["@tf//google/protobuf:protobuf"],
-                   protoc="@tf//google/protobuf:protoc",
-                   default_runtime="@tf//google/protobuf:protobuf",
+                   cc_libs = ["@protobuf//:protobuf"],
+                   protoc="@protobuf//:protoc",
+                   default_runtime="@protobuf//:protobuf",
                    testonly=testonly,
                    visibility=visibility,)
 
@@ -58,8 +53,8 @@ def tf_proto_library_py(name, srcs=[], deps=[], visibility=None, testonly=0):
                    srcs=srcs,
                    srcs_version = "PY2AND3",
                    deps=deps,
-                   default_runtime="@tf//google/protobuf:protobuf_python",
-                   protoc="@tf//google/protobuf:protoc",
+                   default_runtime="@protobuf//:protobuf_python",
+                   protoc="@protobuf//:protoc",
                    visibility=visibility,
                    testonly=testonly,)
 
@@ -72,7 +67,7 @@ def tf_gen_op_libs(op_lib_names):
     native.cc_library(name=n + "_op_lib",
                       copts=tf_copts(),
                       srcs=["ops/" + n + ".cc"],
-                      deps=(["@tf//tensorflow/core:framework"]),
+                      deps=(["@org_tensorflow//tensorflow/core:framework"]),
                       visibility=["//visibility:public"],
                       alwayslink=1,
                       linkstatic=1,)
@@ -89,8 +84,8 @@ def tf_gen_op_wrapper_py(name, out=None, hidden=[], visibility=None, deps=[],
       linkopts = ["-lm"],
       copts = tf_copts(),
       linkstatic = 1,   # Faster to link this one-time-use binary dynamically
-      deps = (["@tf//tensorflow/core:framework",
-               "@tf//tensorflow/python:python_op_gen_main"] + deps),
+      deps = (["@org_tensorflow//tensorflow/core:framework",
+               "@org_tensorflow//tensorflow/python:python_op_gen_main"] + deps),
   )
 
   # Invoke the previous cc_binary to generate a python file.
@@ -110,5 +105,5 @@ def tf_gen_op_wrapper_py(name, out=None, hidden=[], visibility=None, deps=[],
                     srcs_version="PY2AND3",
                     visibility=visibility,
                     deps=[
-                        "@tf//tensorflow/python:framework_for_generated_wrappers",
+                        "@org_tensorflow//tensorflow/python:framework_for_generated_wrappers",
                     ],)

syntaxnet/syntaxnet/term_frequency_map.cc

@@ -58,10 +58,10 @@ void TermFrequencyMap::Load(const string &filename, int min_frequency,
   if (max_num_terms <= 0) max_num_terms = std::numeric_limits<int>::max();
 
   // Read the first line (total # of terms in the mapping).
-  tensorflow::RandomAccessFile *file;
+  std::unique_ptr<tensorflow::RandomAccessFile> file;
   TF_CHECK_OK(tensorflow::Env::Default()->NewRandomAccessFile(filename, &file));
   static const int kInputBufferSize = 1 * 1024 * 1024; /* bytes */
-  tensorflow::io::InputBuffer input(file, kInputBufferSize);
+  tensorflow::io::InputBuffer input(file.get(), kInputBufferSize);
   string line;
   TF_CHECK_OK(input.ReadLine(&line));
   int32 total = -1;
@@ -119,7 +119,7 @@ void TermFrequencyMap::Save(const string &filename) const {
   std::sort(sorted_data.begin(), sorted_data.end(), SortByFrequencyThenTerm());
 
   // Write the number of terms.
-  tensorflow::WritableFile *file;
+  std::unique_ptr<tensorflow::WritableFile> file;
   TF_CHECK_OK(tensorflow::Env::Default()->NewWritableFile(filename, &file));
   CHECK_LE(term_index_.size(), std::numeric_limits<int32>::max());  // overflow
   const int32 num_terms = term_index_.size();
@@ -136,15 +136,14 @@ void TermFrequencyMap::Save(const string &filename) const {
   TF_CHECK_OK(file->Close()) << "for file " << filename;
   LOG(INFO) << "Saved " << term_index_.size() << " terms to " << filename
             << ".";
-  delete file;
 }
 
 TagToCategoryMap::TagToCategoryMap(const string &filename) {
   // Load the mapping.
-  tensorflow::RandomAccessFile *file;
+  std::unique_ptr<tensorflow::RandomAccessFile> file;
   TF_CHECK_OK(tensorflow::Env::Default()->NewRandomAccessFile(filename, &file));
   static const int kInputBufferSize = 1 * 1024 * 1024; /* bytes */
-  tensorflow::io::InputBuffer input(file, kInputBufferSize);
+  tensorflow::io::InputBuffer input(file.get(), kInputBufferSize);
   string line;
   while (input.ReadLine(&line) == tensorflow::Status::OK()) {
     vector<string> pair = utils::Split(line, '\t');
@@ -174,15 +173,14 @@ void TagToCategoryMap::SetCategory(const string &tag, const string &category) {
 
 void TagToCategoryMap::Save(const string &filename) const {
   // Write tag and category on each line.
-  tensorflow::WritableFile *file;
+  std::unique_ptr<tensorflow::WritableFile> file;
   TF_CHECK_OK(tensorflow::Env::Default()->NewWritableFile(filename, &file));
   for (const auto &pair : tag_to_category_) {
     const string line =
         tensorflow::strings::StrCat(pair.first, "\t", pair.second, "\n");
     TF_CHECK_OK(file->Append(line));
   }
   TF_CHECK_OK(file->Close()) << "for file " << filename;
-  delete file;
 }
 
 }  // namespace syntaxnet

syntaxnet/tensorflow

@@ -27,7 +27,7 @@ cc_test(
         "unicodetext_unittest.cc",
     ],
     deps = [
-        "@tf//tensorflow/core:testlib",
+        "@org_tensorflow//tensorflow/core:testlib",
         ":unicodetext",
     ],
 )