Method AsSparkType

Namespace

SparkTest.NET.Extensions

Assembly

SparkTest.NET.dll

AsSparkType(Type)

Converts a type to its spark type

public static DataType AsSparkType(this Type type)

Parameters

type Type

.NET type

Returns

DataType

spark data type

Examples

Simple types,

typeof(string).AsSparkType().Should().Be(new StringType());

typeof(long).AsSparkType().Should().Be(new LongType());

Array like types,

typeof(int[]).AsSparkType().Should().Be(new ArrayType(new IntegerType()));

typeof(IEnumerable<float>).AsSparkType().Should().Be(new ArrayType(new FloatType()));

Map like types,

typeof(Dictionary<int, string>)
    .AsSparkType()
    .Should()
    .Be(new MapType(new IntegerType(), new StringType()));

typeof(IEnumerable<KeyValuePair<long, byte[]>>)
    .AsSparkType()
    .Should()
    .Be(new MapType(new LongType(), new BinaryType()));

Struct like types,

enum Colour
{
    Red,
    Green,
    Blue
}

class Car
{
    public string? Name { get; set; }
    public int Age { get; set; }
    public Colour Colour { get; set; }
}

[Fact(DisplayName = "POCOs can be converted to Spark Struct types, Example 1")]
public static void Case7() =>
    typeof(Car)
        .AsSparkType()
        .Should()
        .Be(
            new StructType(
                new[]
                {
                    new StructField("Name", new StringType()),
                    new StructField("Age", new IntegerType()),
                    new StructField("Colour", new StringType()),
                }
            )
        );

Exceptions

NotSupportedException

if the type is not supported in spark