Creating Arrow Objects

Recipes related to the creation of Arrays, Tables, Tensors and all other Arrow entities.

Create Arrays from Standard C++

Typed subclasses of arrow::ArrayBuilder make it easy to efficiently create Arrow arrays from existing C++ data:

Creating an array from C++ primitives

arrow::Int32Builder builder;
ARROW_RETURN_NOT_OK(builder.Append(1));
ARROW_RETURN_NOT_OK(builder.Append(2));
ARROW_RETURN_NOT_OK(builder.Append(3));
ARROW_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> arr, builder.Finish())
rout << arr->ToString() << std::endl;

Code Output

[
  1,
  2,
  3
]

Note

Builders will allocate data as needed and insertion should have constant amortized time.

Builders can also consume standard C++ containers:

// Raw pointers
arrow::Int64Builder long_builder = arrow::Int64Builder();
std::array<int64_t, 4> values = {1, 2, 3, 4};
ARROW_RETURN_NOT_OK(long_builder.AppendValues(values.data(), values.size()));
ARROW_ASSIGN_OR_RAISE(std::shared_ptr<arrow::Array> arr, long_builder.Finish());
rout << arr->ToString() << std::endl;

// Vectors
arrow::StringBuilder str_builder = arrow::StringBuilder();
std::vector<std::string> strvals = {"x", "y", "z"};
ARROW_RETURN_NOT_OK(str_builder.AppendValues(strvals));
ARROW_ASSIGN_OR_RAISE(arr, str_builder.Finish());
rout << arr->ToString() << std::endl;

// Iterators
arrow::DoubleBuilder dbl_builder = arrow::DoubleBuilder();
std::set<double> dblvals = {1.1, 1.1, 2.3};
ARROW_RETURN_NOT_OK(dbl_builder.AppendValues(dblvals.begin(), dblvals.end()));
ARROW_ASSIGN_OR_RAISE(arr, dbl_builder.Finish());
rout << arr->ToString() << std::endl;

Code Output

[
  1,
  2,
  3,
  4
]
[
  "x",
  "y",
  "z"
]
[
  1.1,
  2.3
]

Note

Builders will not take ownership of data in containers and will make a copy of the underlying data.

Generate Random Data for a Given Schema

To generate random data for a given schema, implementing a type visitor is a good idea. The following example only implements double arrays and list arrays, but could be easily extended to all types.

Using visitor pattern to generate random record batches

class RandomBatchGenerator {
 public:
  std::shared_ptr<arrow::Schema> schema;

  RandomBatchGenerator(std::shared_ptr<arrow::Schema> schema) : schema(schema){};

  arrow::Result<std::shared_ptr<arrow::RecordBatch>> Generate(int32_t num_rows) {
    num_rows_ = num_rows;
    for (std::shared_ptr<arrow::Field> field : schema->fields()) {
      ARROW_RETURN_NOT_OK(arrow::VisitTypeInline(*field->type(), this));
    }

    return arrow::RecordBatch::Make(schema, num_rows, arrays_);
  }

  // Default implementation
  arrow::Status Visit(const arrow::DataType& type) {
    return arrow::Status::NotImplemented("Generating data for", type.ToString());
  }

  arrow::Status Visit(const arrow::DoubleType&) {
    auto builder = arrow::DoubleBuilder();
    std::normal_distribution<> d{/*mean=*/5.0, /*stddev=*/2.0};
    for (int32_t i = 0; i < num_rows_; ++i) {
      ARROW_RETURN_NOT_OK(builder.Append(d(gen_)));
    }
    ARROW_ASSIGN_OR_RAISE(auto array, builder.Finish());
    arrays_.push_back(array);
    return arrow::Status::OK();
  }

  arrow::Status Visit(const arrow::ListType& type) {
    // Generate offsets first, which determines number of values in sub-array
    std::poisson_distribution<> d{/*mean=*/4};
    auto builder = arrow::Int32Builder();
    ARROW_RETURN_NOT_OK(builder.Append(0));
    int32_t last_val = 0;
    for (int32_t i = 0; i < num_rows_; ++i) {
      last_val += d(gen_);
      ARROW_RETURN_NOT_OK(builder.Append(last_val));
    }
    ARROW_ASSIGN_OR_RAISE(auto offsets, builder.Finish());

    // Since children of list has a new length, will use a new generator
    RandomBatchGenerator value_gen(arrow::schema({arrow::field("x", type.value_type())}));
    // Last index from the offsets array becomes the length of the sub-array
    ARROW_ASSIGN_OR_RAISE(auto inner_batch, value_gen.Generate(last_val));
    std::shared_ptr<arrow::Array> values = inner_batch->column(0);

    ARROW_ASSIGN_OR_RAISE(auto array,
                          arrow::ListArray::FromArrays(*offsets.get(), *values.get()));
    arrays_.push_back(array);

    return arrow::Status::OK();
  }

 protected:
  std::random_device rd_{};
  std::mt19937 gen_{rd_()};
  std::vector<std::shared_ptr<arrow::Array>> arrays_;
  int32_t num_rows_;
};  // RandomBatchGenerator

Given such a generator, you can create random test data for any supported schema:

std::shared_ptr<arrow::Schema> schema =
    arrow::schema({arrow::field("x", arrow::float64()),
                   arrow::field("y", arrow::list(arrow::float64()))});

RandomBatchGenerator generator(schema);
ARROW_ASSIGN_OR_RAISE(std::shared_ptr<arrow::RecordBatch> batch, generator.Generate(5));

rout << "Created batch: \n" << batch->ToString();

// Consider using ValidateFull to check correctness
ARROW_RETURN_NOT_OK(batch->ValidateFull());

Code Output

Created batch: 
x:   [
    4.430411238665037,
    1.1402567322890786,
    4.757906450297549,
    7.187500851186735,
    2.9767679068012853
  ]
y:   [
    [
      7.849290083736791,
      7.05431322008432,
      3.417650470075496,
      6.542268012615005,
      2.382945633558648,
      5.190510731957064
    ],
    [
      4.063498588931077,
      8.817756925510391,
      5.3547571198281325,
      6.778059481095653,
      6.530979651948345,
      5.120788794246223
    ],
    [
      6.399947715489706,
      5.02502018174901,
      5.331867600463648
    ],
    [
      6.574170475881674
    ],
    [
      3.204670046727827,
      8.072455895005092,
      2.0748510689797657,
      6.32513834819659,
      4.27734561602461,
      7.1846011751883765
    ]
  ]