irx.record_batch

irx.record_batch

Source: packages/irx/src/irx/record_batch.py

title: Record batch streaming API.

Classes

IrxColumnType(IntEnum)

title: IrxColumnType.

RecordBatchSchema

title: RecordBatchSchema.
attributes:
  _handle:
    type: ctypes.c_void_p
  _lib:
    type: ctypes.CDLL
  _released:
    type: bool
  _col_types:
    type: list[IrxColumnType]
  _elem_types:
    type: list[Optional[IrxColumnType]]
  _struct_fields:
    type: list[Optional[list[tuple[str, IrxColumnType]]]]

Methods

add_field(self, name: str, col_type: IrxColumnType, nullable: bool=True) -> 'RecordBatchSchema'
title: add_field.
parameters:
  name:
    type: str
  col_type:
    type: IrxColumnType
  nullable:
    type: bool
returns:
  type: RecordBatchSchema
add_list_field(self, name: str, elem_type: IrxColumnType, nullable: bool=True) -> 'RecordBatchSchema'
title: Add a list column with the given fixed-width element type.
summary: |-
  elem_type must be one of the fixed-width primitive or temporal types
  (see _LIST_ELEM_CTYPE); bool, utf8, and nested element types are not
  supported yet.
parameters:
  name:
    type: str
  elem_type:
    type: IrxColumnType
  nullable:
    type: bool
returns:
  type: RecordBatchSchema
add_struct_field(self, name: str, fields: Sequence[tuple[str, IrxColumnType]], nullable: bool=True) -> 'RecordBatchSchema'
title: Add a struct column with the given named fixed-width fields.
summary: |-
  Each field is a (name, type) pair whose type must be one of the
  fixed-width primitive or temporal types (see _STRUCT_FIELD_CTYPE);
  bool, utf8, and nested field types are not supported yet.
parameters:
  name:
    type: str
  fields:
    type: Sequence[tuple[str, IrxColumnType]]
  nullable:
    type: bool
returns:
  type: RecordBatchSchema
num_fields(self) -> int
title: Return the number of schema fields.
returns:
  type: int
release(self) -> None
title: Release the underlying schema handle.

RecordBatchBuilder

title: RecordBatchBuilder.
attributes:
  _handle:
    type: ctypes.c_void_p
  _lib:
    type: ctypes.CDLL
  _released:
    type: bool
  _col_types:
    type: list[IrxColumnType]
  _elem_types:
    type: list[Optional[IrxColumnType]]
  _struct_fields:
    type: list[Optional[list[tuple[str, IrxColumnType]]]]

Methods

append_int8(self, col: int, v: int) -> None
title: Append an 8-bit signed integer to a column.
parameters:
  col:
    type: int
  v:
    type: int
append_int16(self, col: int, v: int) -> None
title: Append a 16-bit signed integer to a column.
parameters:
  col:
    type: int
  v:
    type: int
append_int32(self, col: int, v: int) -> None
title: Append a 32-bit signed integer to a column.
parameters:
  col:
    type: int
  v:
    type: int
append_int64(self, col: int, v: int) -> None
title: Append a 64-bit signed integer to a column.
parameters:
  col:
    type: int
  v:
    type: int
append_uint8(self, col: int, v: int) -> None
title: Append an 8-bit unsigned integer to a column.
parameters:
  col:
    type: int
  v:
    type: int
append_uint16(self, col: int, v: int) -> None
title: Append a 16-bit unsigned integer to a column.
parameters:
  col:
    type: int
  v:
    type: int
append_uint32(self, col: int, v: int) -> None
title: Append a 32-bit unsigned integer to a column.
parameters:
  col:
    type: int
  v:
    type: int
append_uint64(self, col: int, v: int) -> None
title: Append a 64-bit unsigned integer to a column.
parameters:
  col:
    type: int
  v:
    type: int
append_float32(self, col: int, v: float) -> None
title: Append a 32-bit floating-point value to a column.
parameters:
  col:
    type: int
  v:
    type: float
append_float64(self, col: int, v: float) -> None
title: Append a 64-bit floating-point value to a column.
parameters:
  col:
    type: int
  v:
    type: float
append_bool(self, col: int, v: bool) -> None
title: Append a boolean value to a column.
parameters:
  col:
    type: int
  v:
    type: bool
append_string(self, col: int, v: str) -> None
title: Append a UTF-8 string to a utf8 or large_utf8 column.
parameters:
  col:
    type: int
  v:
    type: str
append_date(self, col: int, v: date | int) -> None
title: Append a value to a date32 or date64 column.
summary: |-
  Accepts a datetime.date or a raw storage int (days since epoch for
  DATE32, milliseconds since epoch for DATE64).
parameters:
  col:
    type: int
  v:
    type: date | int
append_timestamp(self, col: int, v: datetime | int) -> None
title: Append a value to a timestamp column.
summary: |-
  Accepts a datetime.datetime (naive treated as UTC) or a raw storage
  int already scaled to the column's unit (seconds, milliseconds,
  microseconds, or nanoseconds since epoch).
parameters:
  col:
    type: int
  v:
    type: datetime | int
append_time(self, col: int, v: time | int) -> None
title: Append a value to a time32 or time64 column.
summary: |-
  Accepts a datetime.time or a raw storage int scaled to the column's
  unit (seconds, milliseconds, microseconds, or nanoseconds since
  midnight).
parameters:
  col:
    type: int
  v:
    type: time | int
append_null(self, col: int) -> None
title: Append a null value to a column.
parameters:
  col:
    type: int
append_list(self, col: int, values: Sequence[Any]) -> None
title: Append one list slot to a list column.
summary: |-
  values is a sequence of the column's element type. Temporal elements
  accept datetime.date/datetime/time objects or raw storage ints;
  numeric elements accept plain numbers. Use append_null(col) for a
  null list slot.
parameters:
  col:
    type: int
  values:
    type: Sequence[Any]
append_struct(self, col: int, values: Mapping[str, Any] | Sequence[Any]) -> None
title: Append one struct slot to a struct column.
summary: |-
  values gives one value per field, either as a mapping keyed by field
  name or as a positional sequence in field order. Temporal fields
  accept datetime.date/datetime/time objects or raw storage ints;
  numeric fields accept plain numbers. Use append_null(col) for a null
  struct slot.
parameters:
  col:
    type: int
  values:
    type: Mapping[str, Any] | Sequence[Any]
finish(self) -> 'RecordBatch'
title: finish.
returns:
  type: RecordBatch
release(self) -> None
title: Release the underlying builder handle.

RecordBatch

title: RecordBatch.
summary: |-
  Null-slot behaviour is uniform across every getter: a null value reads
  back as the zero value of its type (0 for numerics, empty string for
  utf8/large_utf8, epoch-relative 0 for date/timestamp/time). Callers
  that need to distinguish a real zero/empty value from a null must
  check ``is_null(col, row)`` first.
attributes:
  _handle:
    type: ctypes.c_void_p
  _lib:
    type: ctypes.CDLL
  _released:
    type: bool

Methods

num_rows(self) -> int
title: Return the number of rows in the batch.
returns:
  type: int
num_columns(self) -> int
title: Return the number of columns in the batch.
returns:
  type: int
get_int8(self, col: int, row: int) -> int
title: Return an 8-bit signed integer value from the batch.
parameters:
  col:
    type: int
  row:
    type: int
returns:
  type: int
get_int16(self, col: int, row: int) -> int
title: Return a 16-bit signed integer value from the batch.
parameters:
  col:
    type: int
  row:
    type: int
returns:
  type: int
get_int32(self, col: int, row: int) -> int
title: Return a 32-bit signed integer value from the batch.
parameters:
  col:
    type: int
  row:
    type: int
returns:
  type: int
get_int64(self, col: int, row: int) -> int
title: Return a 64-bit signed integer value from the batch.
parameters:
  col:
    type: int
  row:
    type: int
returns:
  type: int
get_uint8(self, col: int, row: int) -> int
title: Return an 8-bit unsigned integer value from the batch.
parameters:
  col:
    type: int
  row:
    type: int
returns:
  type: int
get_uint16(self, col: int, row: int) -> int
title: Return a 16-bit unsigned integer value from the batch.
parameters:
  col:
    type: int
  row:
    type: int
returns:
  type: int
get_uint32(self, col: int, row: int) -> int
title: Return a 32-bit unsigned integer value from the batch.
parameters:
  col:
    type: int
  row:
    type: int
returns:
  type: int
get_uint64(self, col: int, row: int) -> int
title: Return a 64-bit unsigned integer value from the batch.
parameters:
  col:
    type: int
  row:
    type: int
returns:
  type: int
get_float32(self, col: int, row: int) -> float
title: Return a 32-bit floating-point value from the batch.
parameters:
  col:
    type: int
  row:
    type: int
returns:
  type: float
get_float64(self, col: int, row: int) -> float
title: Return a 64-bit floating-point value from the batch.
parameters:
  col:
    type: int
  row:
    type: int
returns:
  type: float
get_bool(self, col: int, row: int) -> bool
title: Return a boolean value from the batch.
parameters:
  col:
    type: int
  row:
    type: int
returns:
  type: bool
get_string(self, col: int, row: int) -> str
title: Return a UTF-8 string from a utf8 or large_utf8 column.
summary: |-
  Null slots return an empty string rather than an error, matching
  the behavior of the numeric getters. Callers that need to
  distinguish a real empty string from a null must check
  ``is_null(col, row)`` first.
parameters:
  col:
    type: int
  row:
    type: int
returns:
  type: str
get_date(self, col: int, row: int) -> int
title: Return the raw storage int for a date32 or date64 column.
summary: |-
  Value is days since epoch for DATE32 and milliseconds since epoch
  for DATE64.
parameters:
  col:
    type: int
  row:
    type: int
returns:
  type: int
get_timestamp(self, col: int, row: int) -> int
title: Return the raw storage int for a timestamp column.
summary: |-
  Value is scaled to the column's unit (seconds, milliseconds,
  microseconds, or nanoseconds since epoch).
parameters:
  col:
    type: int
  row:
    type: int
returns:
  type: int
get_time(self, col: int, row: int) -> int
title: Return the raw storage int for a time32 or time64 column.
summary: |-
  Value is scaled to the column's unit (seconds, milliseconds,
  microseconds, or nanoseconds since midnight).
parameters:
  col:
    type: int
  row:
    type: int
returns:
  type: int
get_list(self, col: int, row: int) -> Optional[list[Any]]
title: Return the list at a list-column row, or None if it is null.
summary: |-
  Elements come back as raw storage values: numbers for numeric
  element types and raw storage ints for temporal element types
  (matching the scalar getters). An empty list slot returns an empty
  list; a null list slot returns None.
parameters:
  col:
    type: int
  row:
    type: int
returns:
  type: Optional[list[Any]]
get_struct(self, col: int, row: int) -> Optional[dict[str, Any]]
title: Return the struct at a struct-column row, or None if it is null.
summary: |-
  Fields come back keyed by name; values are raw storage values
  (numbers for numeric fields, raw storage ints for temporal fields,
  matching the scalar getters). A null struct slot returns None.
parameters:
  col:
    type: int
  row:
    type: int
returns:
  type: Optional[dict[str, Any]]
is_null(self, col: int, row: int) -> bool
title: Return whether the value at the supplied location is null.
parameters:
  col:
    type: int
  row:
    type: int
returns:
  type: bool
release(self) -> None
title: Release the underlying batch handle.

RecordBatchStreamWriter

title: RecordBatchStreamWriter.
attributes:
  _handle:
    type: ctypes.c_void_p
  _lib:
    type: ctypes.CDLL
  _is_buffer:
    type: bool
  _closed:
    type: bool
  _released:
    type: bool

Methods

open_file(cls, schema: RecordBatchSchema, path: str | os.PathLike[str]) -> 'RecordBatchStreamWriter'
title: Open a stream writer backed by a file path.
parameters:
  schema:
    type: RecordBatchSchema
  path:
    type: str | os.PathLike[str]
returns:
  type: RecordBatchStreamWriter
open_buffer(cls, schema: RecordBatchSchema) -> 'RecordBatchStreamWriter'
title: Open a stream writer backed by an in-memory buffer.
parameters:
  schema:
    type: RecordBatchSchema
returns:
  type: RecordBatchStreamWriter
write_batch(self, batch: RecordBatch) -> None
title: Write a completed batch to the stream.
parameters:
  batch:
    type: RecordBatch
close(self) -> None
title: Close the underlying stream writer.
buffer_data(self) -> bytes
title: buffer_data.
returns:
  type: bytes
release(self) -> None
title: Release the underlying stream writer handle.

RecordBatchStreamReader

title: RecordBatchStreamReader.
attributes:
  _handle:
    type: ctypes.c_void_p
  _lib:
    type: ctypes.CDLL
  _closed:
    type: bool

Methods

open_file(cls, path: str | os.PathLike[str]) -> 'RecordBatchStreamReader'
title: Open a stream reader backed by a file path.
parameters:
  path:
    type: str | os.PathLike[str]
returns:
  type: RecordBatchStreamReader
open_buffer(cls, data: bytes) -> 'RecordBatchStreamReader'
title: Open a stream reader backed by an in-memory buffer.
parameters:
  data:
    type: bytes
returns:
  type: RecordBatchStreamReader
next_batch(self) -> Optional[RecordBatch]
title: Return the next RecordBatch or None at end-of-stream.
returns:
  type: Optional[RecordBatch]
close(self) -> None
title: Close the underlying stream reader.
Back to top