[C#] Use Arrow scalars

[Platob]

Describe the enhancement requested

Hello,

I would like use more user friendly methods to serialize/deserialize c# objects creating scalars, specifically for nested structures

But i have doubts if this is a good start implementation or completly wrong, any advice or correction ?

Br
Nick

Currently it looks like

Interfaces:

using Apache.Arrow.Types;

namespace Apache.Arrow.Scalars
{
    public interface Scalar
    {
        IArrowType DataType { get; }

        void Validate();

        void ValidateFull();
    }

    public interface Scalar<TArrowType> : Scalar where TArrowType : IArrowType
    {

    }

    public interface Scalar<TArrowType, TValue> : Scalar<TArrowType> where TArrowType : IArrowType
    {
        TValue Value { get; }
    }

    public interface PrimitiveScalar : Scalar
    {

    }

    public interface PrimitiveScalar<TArrowType> : Scalar<TArrowType>, PrimitiveScalar where TArrowType : IArrowType
    {

    }

    public interface PrimitiveScalar<TArrowType, TValue> : Scalar<TArrowType, TValue>, PrimitiveScalar<TArrowType> where TArrowType : IArrowType
    {

    }

    public interface NumericScalar : PrimitiveScalar
    {

    }

    public interface NumericScalar<TArrowType, TValue> : PrimitiveScalar<TArrowType, TValue>, NumericScalar where TArrowType : IArrowType
    {

    }

    public interface BinaryScalarBase : PrimitiveScalar
    {

    }

    public interface BinaryScalarBase<TArrowType> : PrimitiveScalar<TArrowType>, BinaryScalarBase where TArrowType : IArrowType
    {

    }

    public interface BinaryScalarBase<TArrowType, TValue> : PrimitiveScalar<TArrowType, TValue>, BinaryScalarBase<TArrowType> where TArrowType : IArrowType
    {

    }

    public interface TemporalScalar : PrimitiveScalar
    {

    }

    public interface TemporalScalar<TArrowType> : PrimitiveScalar<TArrowType>, TemporalScalar where TArrowType : IArrowType
    {

    }

    public interface TemporalScalar<TArrowType, TValue> : PrimitiveScalar<TArrowType, TValue>, TemporalScalar<TArrowType> where TArrowType : IArrowType
    {

    }

    public interface ListScalarBase : Scalar
    {

    }

    public interface ListScalarBase<TArrowType> : Scalar<TArrowType>, ListScalarBase where TArrowType : IArrowType
    {

    }

    public interface ListScalarBase<TArrowType, TValue> : Scalar<TArrowType, TValue>, ListScalarBase<TArrowType> where TArrowType : IArrowType
    {

    }
}

Scalar structs:

using System;
using Apache.Arrow.Types;

namespace Apache.Arrow.Scalars
{
    public readonly struct NullScalar : Scalar
    {
        public IArrowType DataType => NullType.Default;

        public void Validate()
        {

        }

        public void ValidateFull()
        {

        }
    }

    public readonly struct BooleanScalar : PrimitiveScalar<BooleanType, bool>
    {
        public IArrowType DataType => BooleanType.Default;

        public bool Value { get; }

        public BooleanScalar(bool value)
        {
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Boolean)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(bool) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    public readonly struct Int8Scalar : NumericScalar<Int8Type, sbyte>
    {
        public IArrowType DataType => Int8Type.Default;

        public sbyte Value { get; }

        public Int8Scalar(sbyte value)
        {
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Int8)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(sbyte) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    public readonly struct UInt8Scalar : NumericScalar<UInt8Type, byte>
    {
        public IArrowType DataType => UInt8Type.Default;

        public byte Value { get; }

        public UInt8Scalar(byte value)
        {
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.UInt8)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(byte) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    public readonly struct Int16Scalar : NumericScalar<Int16Type, short>
    {
        public IArrowType DataType => Int16Type.Default;

        public short Value { get; }

        public Int16Scalar(short value)
        {
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Int16)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(short) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    public readonly struct UInt16Scalar : NumericScalar<UInt16Type, ushort>
    {
        public IArrowType DataType => UInt16Type.Default;

        public ushort Value { get; }

        public UInt16Scalar(ushort value)
        {
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.UInt16)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(ushort) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    public readonly struct Int32Scalar : NumericScalar<Int32Type, int>
    {
        public IArrowType DataType => Int32Type.Default;

        public int Value { get; }

        public Int32Scalar(int value)
        {
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Int32)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(int) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    public readonly struct UInt32Scalar : NumericScalar<UInt32Type, uint>
    {
        public IArrowType DataType => UInt32Type.Default;

        public uint Value { get; }

        public UInt32Scalar(uint value)
        {
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.UInt32)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(uint) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    public readonly struct Int64Scalar : NumericScalar<Int64Type, long>
    {
        public IArrowType DataType => Int64Type.Default;

        public long Value { get; }

        public Int64Scalar(long value)
        {
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Int64)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(long) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    public readonly struct UInt64Scalar : NumericScalar<UInt64Type, ulong>
    {
        public IArrowType DataType => UInt64Type.Default;

        public ulong Value { get; }

        public UInt64Scalar(ulong value)
        {
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.UInt64)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(ulong) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

# if NET6_0_OR_GREATER
    public readonly struct HalfFloatScalar : NumericScalar<HalfFloatType, Half>
    {
        public IArrowType DataType => HalfFloatType.Default;

        public Half Value { get; }

        public HalfFloatScalar(Half value)
        {
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.HalfFloat)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(float) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }
# endif

    public readonly struct FloatScalar : NumericScalar<FloatType, float>
    {
        public IArrowType DataType => FloatType.Default;

        public float Value { get; }

        public FloatScalar(float value)
        {
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Float)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(float) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    public readonly struct DoubleScalar : NumericScalar<DoubleType, double>
    {
        public IArrowType DataType => DoubleType.Default;

        public double Value { get; }

        public DoubleScalar(double value)
        {
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Double)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(double) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    // Binary
    public readonly struct BinaryScalar : BinaryScalarBase<BinaryType, byte[]>
    {
        public IArrowType DataType => BinaryType.Default;

        public byte[] Value { get; }

        public BinaryScalar(byte[] value)
        {
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Binary)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(byte[]) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    // String
    public readonly struct StringScalar : BinaryScalarBase<StringType, string>
    {
        public IArrowType DataType => StringType.Default;

        public string Value { get; }

        public StringScalar(string value)
        {
            Value = value;
        }

        public void Validate()
        {
            if (Value == null)
                throw new InvalidOperationException("Invalid csharp null value for string scalar");

            if (DataType.TypeId != ArrowTypeId.String)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(string) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    // FixedSizedBinary
    public readonly struct FixedSizedBinaryScalar : BinaryScalarBase<FixedSizeBinaryType, byte[]>
    {
        public IArrowType DataType { get; }

        public byte[] Value { get; }

        public FixedSizedBinaryScalar(byte[] value, FixedSizeBinaryType dataType)
        {
            DataType = dataType;
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.FixedSizedBinary)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(byte[]) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    // Date32
    public readonly struct Date32Scalar : TemporalScalar<Date32Type, DateTime>
    {
        public IArrowType DataType => Date32Type.Default;

        public DateTime Value { get; }

        public Date32Scalar(DateTime value)
        {
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Date32)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(int) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    // Date64
    public readonly struct Date64Scalar : TemporalScalar<Date64Type, DateTime>
    {
        public IArrowType DataType => Date64Type.Default;

        public DateTime Value { get; }

        public Date64Scalar(DateTime value)
        {
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Date64)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(long) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    // Timestamp
    public readonly struct TimestampScalar : TemporalScalar<TimestampType, DateTime>
    {
        public IArrowType DataType { get; }

        public DateTime Value { get; }

        public TimestampScalar(DateTime value, TimestampType dataType)
        {
            DataType = dataType;
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Timestamp)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(long) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    // Time32
    public readonly struct Time32Scalar : TemporalScalar<Time32Type, TimeSpan>
    {
        public IArrowType DataType { get; }

        public TimeSpan Value { get; }

        public Time32Scalar(TimeSpan value, Time32Type dataType)
        {
            DataType = dataType;
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Time32)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(int) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    // Time64
    public readonly struct Time64Scalar : TemporalScalar<Time64Type, TimeSpan>
    {
        public IArrowType DataType { get; }

        public TimeSpan Value { get; }

        public Time64Scalar(TimeSpan value, Time64Type dataType)
        {
            DataType = dataType;
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Time64)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(long) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    // Interval
    public readonly struct IntervalScalar : TemporalScalar<IntervalType, TimeSpan>
    {
        public IArrowType DataType { get; }

        public TimeSpan Value { get; }

        public IntervalScalar(TimeSpan value, IntervalType dataType)
        {
            DataType = dataType;
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Interval)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(long) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    // Decimal128
    public readonly struct Decimal128Scalar : PrimitiveScalar<Decimal128Type, decimal>
    {
        public IArrowType DataType { get; }

        public decimal Value { get; }

        public Decimal128Scalar(decimal value, Decimal128Type dataType)
        {
            DataType = dataType;
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Decimal128)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(decimal) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    // Decimal256
    public readonly struct Decimal256Scalar : PrimitiveScalar<Decimal256Type, decimal>
    {
        public IArrowType DataType { get; }

        public decimal Value { get; }

        public Decimal256Scalar(decimal value, Decimal256Type dataType)
        {
            DataType = dataType;
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Decimal256)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(decimal) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    // Duration
    public readonly struct DurationScalar : TemporalScalar<DurationType, TimeSpan>
    {
        public IArrowType DataType { get; }

        public TimeSpan Value { get; }

        public DurationScalar(TimeSpan value, DurationType dataType)
        {
            DataType = dataType;
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Duration)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(long) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    // Decimal32
    public readonly struct Decimal32Scalar : PrimitiveScalar<Decimal32Type, decimal>
    {
        public IArrowType DataType { get; }

        public decimal Value { get; }

        public Decimal32Scalar(decimal value, Decimal32Type dataType)
        {
            DataType = dataType;
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Decimal32)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(decimal) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    // Decimal64
    public readonly struct Decimal64Scalar : PrimitiveScalar<Decimal64Type, decimal>
    {
        public IArrowType DataType { get; }

        public decimal Value { get; }

        public Decimal64Scalar(decimal value, Decimal64Type dataType)
        {
            DataType = dataType;
            Value = value;
        }

        public void Validate()
        {
            if (DataType.TypeId != ArrowTypeId.Decimal64)
                throw new InvalidOperationException("Invalid arrow type for scalar");

            if (typeof(decimal) != Value.GetType())
                throw new InvalidOperationException("Invalid csharp value type for scalar");
        }

        public void ValidateFull()
        {
            Validate();
        }
    }

    public readonly struct ListScalar : ListScalarBase<ListType>
    {
        public IArrowType DataType => Values.Data.DataType;

        public IArrowArray Values { get; }

        public ListScalar(IArrowArray values)
        {
            Values = values;
        }

        public void Validate()
        {

        }

        public void ValidateFull()
        {

        }
    }

    public readonly struct StructScalar : Scalar<StructType>
    {
        public IArrowType DataType { get; }

        public Scalar[] Values { get; }

        public StructScalar(Scalar[] values, StructType dataType)
        {
            DataType = dataType;
            Values = values;
        }

        public void Validate()
        {
            foreach (var value in Values)
            {
                value.Validate();
            }
        }

        public void ValidateFull()
        {
            foreach (var value in Values)
            {
                value.ValidateFull();
            }
        }
    }
}

Component(s)

C#

[CurtHagenlocher]

I'm not convinced that this approach leads someplace valuable, but I'm willing to be convinced. What I think would be most helpful is to have something like the serialization support for System.Text.Json where you can autogenerate code to translate or wrap between Arrow arrays and user-defined C# types. That request is reflected as apache/arrow-dotnet#186.

Closing this in favor of the generic issue apache/arrow-dotnet#181 in the new repository.