Merge branch 'apache:main' into Pyarrow_WoA

Author: MugundanMCW

.github/workflows/ruby.yml

@@ -124,7 +124,7 @@ jobs:
         run: archery docker push ubuntu-ruby
 
   macos:
-    name: ARM64 macOS 14 GLib & Ruby
+    name: ARM64 macOS GLib & Ruby
     runs-on: macos-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 60

c_glib/arrow-glib/reader.cpp

@@ -668,6 +668,35 @@ garrow_record_batch_file_reader_read_record_batch(GArrowRecordBatchFileReader *r
   }
 }
 
+/**
+ * garrow_record_batch_file_reader_get_metadata:
+ * @reader: A #GArrowRecordBatchFileReader.
+ *
+ * Returns: (nullable) (element-type utf8 utf8) (transfer full):
+ *   The metadata in the footer.
+ *
+ * Since: 24.0.0
+ */
+GHashTable *
+garrow_record_batch_file_reader_get_metadata(GArrowRecordBatchFileReader *reader)
+{
+  auto arrow_reader = garrow_record_batch_file_reader_get_raw(reader);
+  auto arrow_metadata = arrow_reader->metadata();
+
+  if (!arrow_metadata) {
+    return nullptr;
+  }
+
+  auto metadata = g_hash_table_new(g_str_hash, g_str_equal);
+  const auto n = arrow_metadata->size();
+  for (int64_t i = 0; i < n; ++i) {
+    g_hash_table_insert(metadata,
+                        const_cast<gchar *>(arrow_metadata->key(i).c_str()),
+                        const_cast<gchar *>(arrow_metadata->value(i).c_str()));
+  }
+  return metadata;
+}
+
 struct GArrowFeatherFileReaderPrivate
 {
   std::shared_ptr<arrow::ipc::feather::Reader> feather_reader;

c_glib/arrow-glib/reader.h

@@ -166,6 +166,10 @@ garrow_record_batch_file_reader_read_record_batch(GArrowRecordBatchFileReader *r
                                                   guint i,
                                                   GError **error);
 
+GARROW_AVAILABLE_IN_24_0
+GHashTable *
+garrow_record_batch_file_reader_get_metadata(GArrowRecordBatchFileReader *reader);
+
 #define GARROW_TYPE_FEATHER_FILE_READER (garrow_feather_file_reader_get_type())
 GARROW_AVAILABLE_IN_ALL
 G_DECLARE_DERIVABLE_TYPE(GArrowFeatherFileReader,

c_glib/arrow-glib/writer.cpp

@@ -20,6 +20,8 @@
 #include <arrow-glib/array.hpp>
 #include <arrow-glib/enums.h>
 #include <arrow-glib/error.hpp>
+#include <arrow-glib/internal-hash-table.hpp>
+#include <arrow-glib/ipc-options.hpp>
 #include <arrow-glib/record-batch.hpp>
 #include <arrow-glib/schema.hpp>
 #include <arrow-glib/table.hpp>
@@ -288,16 +290,50 @@ GArrowRecordBatchFileWriter *
 garrow_record_batch_file_writer_new(GArrowOutputStream *sink,
                                     GArrowSchema *schema,
                                     GError **error)
+{
+  return garrow_record_batch_file_writer_new_full(sink, schema, nullptr, nullptr, error);
+}
+
+/**
+ * garrow_record_batch_file_writer_new_full:
+ * @sink: The output of the writer.
+ * @schema: The schema of the writer.
+ * @options: (nullable): The options for serialization.
+ * @metadata: (nullable) (element-type utf8 utf8): The custom metadata in
+ *   the footer.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: (nullable): A newly created #GArrowRecordBatchFileWriter
+ *   or %NULL on error.
+ *
+ * Since: 24.0.0
+ */
+GArrowRecordBatchFileWriter *
+garrow_record_batch_file_writer_new_full(GArrowOutputStream *sink,
+                                         GArrowSchema *schema,
+                                         GArrowWriteOptions *options,
+                                         GHashTable *metadata,
+                                         GError **error)
 {
   auto arrow_sink = garrow_output_stream_get_raw(sink);
   auto arrow_schema = garrow_schema_get_raw(schema);
+  arrow::ipc::IpcWriteOptions arrow_options = arrow::ipc::IpcWriteOptions::Defaults();
+  if (options) {
+    arrow_options = *garrow_write_options_get_raw(options);
+  }
+  std::shared_ptr<arrow::KeyValueMetadata> arrow_metadata;
+  if (metadata) {
+    arrow_metadata = garrow_internal_hash_table_to_metadata(metadata);
+  }
+
   std::shared_ptr<arrow::ipc::RecordBatchWriter> arrow_writer;
-  auto arrow_writer_result = arrow::ipc::MakeFileWriter(arrow_sink, arrow_schema);
-  if (garrow::check(error, arrow_writer_result, "[record-batch-file-writer][open]")) {
+  auto arrow_writer_result =
+    arrow::ipc::MakeFileWriter(arrow_sink, arrow_schema, arrow_options, arrow_metadata);
+  if (garrow::check(error, arrow_writer_result, "[record-batch-file-writer][new]")) {
     auto arrow_writer = *arrow_writer_result;
     return garrow_record_batch_file_writer_new_raw(&arrow_writer);
   } else {
-    return NULL;
+    return nullptr;
   }
 }
 

c_glib/arrow-glib/writer.h

@@ -20,6 +20,7 @@
 #pragma once
 
 #include <arrow-glib/array.h>
+#include <arrow-glib/ipc-options.h>
 #include <arrow-glib/record-batch.h>
 #include <arrow-glib/schema.h>
 
@@ -94,6 +95,14 @@ garrow_record_batch_file_writer_new(GArrowOutputStream *sink,
                                     GArrowSchema *schema,
                                     GError **error);
 
+GARROW_AVAILABLE_IN_24_0
+GArrowRecordBatchFileWriter *
+garrow_record_batch_file_writer_new_full(GArrowOutputStream *sink,
+                                         GArrowSchema *schema,
+                                         GArrowWriteOptions *options,
+                                         GHashTable *metadata,
+                                         GError **error);
+
 /**
  * GArrowCSVQuotingStyle:
  * @GARROW_CSV_QUOTING_STYLE_NEEDED: Only enclose values in quotes which need them.

c_glib/test/test-file-writer.rb

@@ -88,4 +88,36 @@ def test_write_table
       input.close
     end
   end
+
+  def test_footer_custom_metadata
+    tempfile = Tempfile.open("arrow-ipc-file-writer")
+    output = Arrow::FileOutputStream.new(tempfile.path, false)
+
+    array = build_boolean_array([true, false, true])
+    field = Arrow::Field.new("enabled", Arrow::BooleanDataType.new)
+    schema = Arrow::Schema.new([field])
+
+    options = Arrow::WriteOptions.new
+    metadata = {"key1" => "value1", "key2" => "value2"}
+    begin
+      file_writer = Arrow::RecordBatchFileWriter.new(output,
+                                                     schema,
+                                                     options,
+                                                     metadata)
+      file_writer.close
+      assert do
+        file_writer.closed?
+      end
+    ensure
+      output.close
+    end
+
+    input = Arrow::MemoryMappedInputStream.new(tempfile.path)
+    begin
+      file_reader = Arrow::RecordBatchFileReader.new(input)
+      assert_equal(metadata, file_reader.metadata)
+    ensure
+      input.close
+    end
+  end
 end

cpp/CMakeLists.txt

@@ -115,6 +115,8 @@ list(PREPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules")
 # vcpkg (ab)uses CMAKE_TOOLCHAIN_FILE to inject its logic into CMake
 if(ARROW_DEPENDENCY_SOURCE STREQUAL "VCPKG")
   include(Usevcpkg)
+elseif(ARROW_PACKAGE_KIND STREQUAL "vcpkg")
+  set(ARROW_VCPKG ON)
 endif()
 
 project(arrow VERSION "${ARROW_BASE_VERSION}")

cpp/cmake_modules/FindSnappyAlt.cmake

@@ -52,9 +52,12 @@ if(Snappy_FOUND)
       set(SnappyAlt_FOUND TRUE)
       return()
     else()
-      # The Conan's Snappy package always uses Snappy::snappy and it's
-      # an INTERFACE_LIBRARY.
-      get_target_property(Snappy Snappy::snappy TYPE)
+      # The Snappy packages in Conan and vcpkg always use
+      # Snappy::snappy.
+      #
+      # The Snappy package in Conan use an INTERFACE_LIBRARY for
+      # Snappy::snappy.
+      get_target_property(Snappy_TYPE Snappy::snappy TYPE)
       if(Snappy_TYPE STREQUAL "STATIC_LIBRARY" OR Snappy_TYPE STREQUAL
                                                   "INTERFACE_LIBRARY")
         set(Snappy_TARGET Snappy::snappy)

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -1683,7 +1683,8 @@ endif()
 if(ARROW_BUILD_TESTS
    OR ARROW_BUILD_BENCHMARKS
    OR ARROW_BUILD_INTEGRATION
-   OR ARROW_USE_GLOG)
+   OR ARROW_USE_GLOG
+   OR (ARROW_FLIGHT_SQL AND ARROW_BUILD_EXAMPLES))
   set(ARROW_NEED_GFLAGS TRUE)
 else()
   set(ARROW_NEED_GFLAGS FALSE)

cpp/src/arrow/util/align_util.h

@@ -46,7 +46,8 @@ inline BitmapWordAlignParams BitmapWordAlign(const uint8_t* data, int64_t bit_of
                                              int64_t length) {
   // TODO: We can remove this condition once CRAN upgrades its macOS
   // SDK from 11.3.
-#if defined(__clang__) && !defined(__cpp_lib_bitops) && !defined(__EMSCRIPTEN__)
+  // __apple_build_version__ should be defined only on Apple clang
+#if defined(__apple_build_version__) && !defined(__cpp_lib_bitops)
   static_assert((ALIGN_IN_BYTES != 0) && ((ALIGN_IN_BYTES & (ALIGN_IN_BYTES - 1)) == 0),
                 "ALIGN_IN_BYTES should be a positive power of two");
 #else