Class: HDF5::Dataset

Inherits:

Object

• Object
• HDF5::Dataset

Defined in:

lib/hdf5/dataset.rb

Class Method Summary

• .create(parent_id, name, data = nil, shape: nil, dtype: nil, maxshape: nil, chunks: nil, compression: nil, compression_opts: nil, shuffle: false, fletcher32: false, fillvalue: nil, context: nil, casting: :safe) ⇒ Object
• .open(parent_id, name, context: nil) ⇒ Object

Instance Method Summary

• #[](*selection) ⇒ Object
• #[]=(*selection, value) ⇒ Object
• #append(data, axis: 0) ⇒ Object
• #attrs ⇒ Object
• #chunks ⇒ Object
• #close ⇒ Object
• #closed? ⇒ Boolean
• #dtype ⇒ Object
• #each_block(max_bytes:) ⇒ Object
• #each_chunk ⇒ Object
• #fillvalue ⇒ Object
• #initialize(parent_id, name) ⇒ Dataset constructor

  A new instance of Dataset.

• #maxshape ⇒ Object
• #ndim ⇒ Object
• #read(selection: nil, dtype: nil, casting: :safe) ⇒ Object
• #read_array(selection: nil, flatten: false, dtype: nil, casting: :safe) ⇒ Object
• #read_into(destination, selection: nil, casting: :safe) ⇒ Object
• #resize(new_shape) ⇒ Object
• #shape ⇒ Object
• #size ⇒ Object
• #write(data, selection: nil, casting: :safe) ⇒ Object

Constructor Details

#initialize(parent_id, name) ⇒ Dataset

Returns a new instance of Dataset.

# File 'lib/hdf5/dataset.rb', line 238

def initialize(parent_id, name)
  initialize_from_id(HDF5::FFI.H5Dopen2(parent_id, name, HDF5::DEFAULT_PROPERTY_LIST), name, nil)
end

Class Method Details

.create(parent_id, name, data = nil, shape: nil, dtype: nil, maxshape: nil, chunks: nil, compression: nil, compression_opts: nil, shuffle: false, fletcher32: false, fillvalue: nil, context: nil, casting: :safe) ⇒ Object

# File 'lib/hdf5/dataset.rb', line 4

def create(parent_id, name, data = nil, shape: nil, dtype: nil, maxshape: nil, chunks: nil, compression: nil,
           compression_opts: nil, shuffle: false, fletcher32: false, fillvalue: nil, context: nil, casting: :safe)
  DataHelpers.validate_casting!(casting)

  raise ArgumentError, 'shape: and dtype: are required when data: is omitted' if data.nil? && (!shape || !dtype)

  empty_data = data.is_a?(HDF5::Empty)
  explicit_dtype = DType.for_symbol(dtype) if dtype
  if empty_data
    raise ShapeError, 'Null datasets cannot have a shape' unless shape.nil?
    if explicit_dtype && explicit_dtype.to_sym != data.dtype.to_sym
      raise ConversionError, 'dtype must match the Null dataset dtype'
    end
  end
  inferred_string = HDF5::StringCodec.string_data?(data)
  if inferred_string && explicit_dtype && explicit_dtype.kind != :string
    raise ConversionError, 'Cannot create a numeric dataset from string data'
  end
  string_type = inferred_string || explicit_dtype&.kind == :string || empty_data && data.dtype.kind == :string
  string_data = string_type && !data.nil? && !empty_data
  _string_values, string_shape = HDF5::StringCodec.normalize_data(data) if string_data
  unless data.nil? || string_data || empty_data
    narray = HDF5::DataHelpers.normalize_data(data,
                                              label: 'Dataset data', dtype: explicit_dtype, casting:,
                                              convert: false)
  end
  unless string_type
    dtype_object = if empty_data
                     data.dtype
                   else
                     (explicit_dtype || DType.for_numo(narray))
                   end
  end
  type_id = string_type ? HDF5::StringCodec.datatype_id : dtype_object.storage_type_id
  shape = string_data ? string_shape : narray.shape if shape.nil? && !data.nil? && !empty_data
  raise ShapeError, 'Dataset shape must match data shape' if narray && shape != narray.shape
  raise ShapeError, 'Dataset shape must match string data shape' if string_data && shape != string_shape

  storage_options = maxshape || chunks || compression || compression_opts || shuffle || fletcher32 ||
                    !fillvalue.nil?
  if empty_data && storage_options
    raise UnsupportedFeatureError, 'Null datasets cannot have storage options'
  end

  validate_shape(shape) unless empty_data
  validate_maxshape(maxshape, shape) if maxshape
  chunks = :auto if maxshape && chunks.nil?
  dataspace_id = create_dataspace(shape, maxshape)
  raise NativeError, "Failed to create dataspace for dataset: #{name}" if dataspace_id < 0

  dcpl_id = create_property_list(shape, dtype_object, chunks:, compression:, compression_opts:, shuffle:, fletcher32:,
                                                      fillvalue:, casting:)

  dataset = from_id(
    HDF5::FFI.H5Dcreate2(parent_id, name, type_id, dataspace_id, HDF5::DEFAULT_PROPERTY_LIST,
                         dcpl_id || HDF5::DEFAULT_PROPERTY_LIST, HDF5::DEFAULT_PROPERTY_LIST), name, context
  )
  dataset.write(data) if string_data
  dataset.write(narray) if narray
  initialized = true
  return dataset unless block_given?

  begin
    yield dataset
  ensure
    Native.close_object(dataset)
  end
rescue StandardError
  if dataset && !initialized
    Native.close_object(dataset) unless dataset.closed?
    HDF5::FFI.H5Ldelete(parent_id, name, HDF5::DEFAULT_PROPERTY_LIST)
  end
  raise
ensure
  Native.close([:H5Tclose, string_type ? type_id : nil], [:H5Pclose, dcpl_id], [:H5Sclose, dataspace_id])
end

.open(parent_id, name, context: nil) ⇒ Object

# File 'lib/hdf5/dataset.rb', line 81

def open(parent_id, name, context: nil)
  dataset = from_id(HDF5::FFI.H5Dopen2(parent_id, name, HDF5::DEFAULT_PROPERTY_LIST), name, context)
  return dataset unless block_given?

  begin
    yield dataset
  ensure
    Native.close_object(dataset)
  end
end

Instance Method Details

#[](*selection) ⇒ Object

# File 'lib/hdf5/dataset.rb', line 508

def [](*selection)
  read(selection: selection)
end

#[]=(*selection, value) ⇒ Object

# File 'lib/hdf5/dataset.rb', line 512

def []=(*selection, value)
  write(value, selection: selection)
end

#append(data, axis: 0) ⇒ Object

# File 'lib/hdf5/dataset.rb', line 402

def append(data, axis: 0)
  ensure_open!
  values = HDF5::DataHelpers.normalize_data(data, label: 'Dataset data', dtype: dtype, convert: false)
  current_shape = shape
  raise ShapeError, 'Cannot append to a Null dataset' if current_shape.nil?
  raise ShapeError, 'Cannot append to a scalar dataset' if current_shape.empty?
  unless axis.is_a?(Integer) && axis.between?(0, current_shape.length - 1)
    raise IndexError, "Invalid append axis: #{axis}"
  end
  raise ShapeError, 'Appended data rank must match dataset rank' unless values.shape.length == current_shape.length
  raise ShapeError, 'Appended data shape must match all non-appended dimensions' unless
    values.shape.each_with_index.all? { |dimension, index| index == axis || dimension == current_shape[index] }

  source_dtype = DType.for_numo(values)
  target_dtype = dtype
  raise ConversionError, "Cannot safely cast #{source_dtype.to_sym} to #{target_dtype.to_sym}" unless
    source_dtype.castable_to?(target_dtype)
  return self if values.shape[axis].zero?

  new_shape = current_shape.dup
  new_shape[axis] += values.shape[axis]
  resize(new_shape)
  extent_changed = true
  selection = current_shape.each_with_index.map do |dimension, index|
    index == axis ? dimension...new_shape[index] : 0...dimension
  end
  write(values, selection: selection)
  self
rescue StandardError => e
  raise unless extent_changed

  begin
    resize(current_shape)
  rescue StandardError => rollback_error
    raise NativeError,
          "Append failed (#{e.message}) and extent rollback failed (#{rollback_error.message})", cause: e
  end
  raise e
end

#attrs ⇒ Object

# File 'lib/hdf5/dataset.rb', line 242

def attrs
  ensure_open!
  @attrs ||= AttributeManager.new(@dataset_id, @context)
end

#chunks ⇒ Object

# File 'lib/hdf5/dataset.rb', line 323

def chunks
  ensure_open!
  property_list_id = HDF5::FFI.H5Dget_create_plist(@dataset_id)
  raise NativeError, 'Failed to get dataset creation properties' if property_list_id < 0

  layout = HDF5::FFI.H5Pget_layout(property_list_id)
  raise NativeError, 'Failed to get dataset layout' if layout == :H5D_LAYOUT_ERROR
  return nil unless layout == :H5D_CHUNKED

  dimensions = ::FFI::MemoryPointer.new(:ulong_long, shape.length)
  rank = HDF5::FFI.H5Pget_chunk(property_list_id, shape.length, dimensions)
  raise NativeError, 'Failed to get chunk dimensions' if rank < 0

  dimensions.read_array_of_uint64(rank)
ensure
  Native.close([:H5Pclose, property_list_id])
end

#close ⇒ Object

# File 'lib/hdf5/dataset.rb', line 273

def close
  return if @dataset_id.nil?

  if @context
    @context.close(@dataset_id)
  else
    Native.check(HDF5::FFI.H5Dclose(@dataset_id), 'Failed to close HDF5 dataset')
  end
  @dataset_id = nil
end

#closed? ⇒ Boolean

Returns:

• (Boolean)

# File 'lib/hdf5/dataset.rb', line 284

def closed?
  @dataset_id.nil? || (@context && @context.closed?)
end

#dtype ⇒ Object

# File 'lib/hdf5/dataset.rb', line 288

def dtype
  ensure_open!
  datatype_id = HDF5::FFI.H5Dget_type(@dataset_id)
  raise NativeError, 'Failed to get datatype' if datatype_id < 0

  DType.for_hdf5(datatype_id)
ensure
  Native.close([:H5Tclose, datatype_id])
end

#each_block(max_bytes:) ⇒ Object

Raises:

• (ArgumentError)

# File 'lib/hdf5/dataset.rb', line 535

def each_block(max_bytes:)
  return enum_for(__method__, max_bytes:) unless block_given?

  ensure_open!
  raise ArgumentError, 'max_bytes must be a positive integer' unless max_bytes.is_a?(Integer) && max_bytes.positive?

  current_shape, current_dtype = HDF5::FFI::CALL_LOCK.synchronize { [shape, dtype] }
  raise ShapeError, 'Cannot iterate over a Null dataset' if current_shape.nil?
  if current_dtype.kind == :string
    raise UnsupportedFeatureError, 'each_block cannot bound the byte size of variable-length strings'
  end
  raise ArgumentError, 'max_bytes is smaller than one dataset element' if max_bytes < current_dtype.itemsize

  if current_shape.empty?
    yield [], read
    return
  end
  return if current_shape.any?(&:zero?)

  block_shape = block_shape_for(current_shape, max_bytes / current_dtype.itemsize)
  each_block_selection(current_shape, block_shape) do |selection|
    yield selection, read(selection: selection)
  end
end

#each_chunk ⇒ Object

Raises:

• (UnsupportedFeatureError)

# File 'lib/hdf5/dataset.rb', line 560

def each_chunk
  return enum_for(__method__) unless block_given?

  ensure_open!

  current_shape, chunk_shape = HDF5::FFI::CALL_LOCK.synchronize { [shape, chunks] }
  raise UnsupportedFeatureError, 'each_chunk requires a chunked dataset' unless chunk_shape

  return if current_shape.any?(&:zero?)

  each_block_selection(current_shape, chunk_shape) do |selection|
    yield selection, read(selection: selection)
  end
end

#fillvalue ⇒ Object

# File 'lib/hdf5/dataset.rb', line 360

def fillvalue
  ensure_open!
  dtype_object = dtype
  if dtype_object.kind == :string
    raise UnsupportedFeatureError, 'fillvalue is not supported for string datasets'
  end
  property_list_id = HDF5::FFI.H5Dget_create_plist(@dataset_id)
  raise NativeError, 'Failed to get dataset creation properties' if property_list_id < 0

  buffer = ::FFI::MemoryPointer.new(:char, dtype_object.itemsize)
  status = HDF5::FFI.H5Pget_fill_value(property_list_id, dtype_object.memory_type_id, buffer)
  raise NativeError, 'Failed to get dataset fill value' if status < 0

  value = HDF5::DataHelpers.from_binary(dtype_object, buffer.read_bytes(dtype_object.itemsize), []).extract
  dtype_object.kind == :bool ? !value.zero? : value
ensure
  Native.close([:H5Pclose, property_list_id])
end

#maxshape ⇒ Object

# File 'lib/hdf5/dataset.rb', line 341

def maxshape
  ensure_open!
  dataspace_id = HDF5::FFI.H5Dget_space(@dataset_id)
  raise NativeError, 'Failed to get dataset dataspace' if dataspace_id < 0

  rank = HDF5::FFI.H5Sget_simple_extent_ndims(dataspace_id)
  Native.check(rank, 'Failed to get dataset rank')
  return [] if rank.zero?

  maximums = ::FFI::MemoryPointer.new(:ulong_long, rank)
  status = HDF5::FFI.H5Sget_simple_extent_dims(dataspace_id, nil, maximums)
  raise NativeError, 'Failed to get dataset maximum shape' if status < 0

  unlimited = (1 << (::FFI.type_size(:ulong_long) * 8)) - 1
  maximums.read_array_of_uint64(rank).map { |dimension| dimension == unlimited ? nil : dimension }
ensure
  Native.close([:H5Sclose, dataspace_id])
end

#ndim ⇒ Object

# File 'lib/hdf5/dataset.rb', line 315

def ndim
  shape&.length
end

#read(selection: nil, dtype: nil, casting: :safe) ⇒ Object

# File 'lib/hdf5/dataset.rb', line 442

def read(selection: nil, dtype: nil, casting: :safe)
  ensure_open!
  DataHelpers.validate_casting!(casting)
  type_id = HDF5::FFI.H5Dget_type(@dataset_id)
  raise NativeError, 'Failed to get dataset datatype' if type_id < 0

  current_shape = shape
  if current_shape.nil?
    raise ShapeError, 'Null datasets cannot be sliced' unless selection.nil?

    current_dtype = dtype ? DType.for_symbol(dtype) : DType.for_hdf5(type_id)
    unless DType.for_hdf5(type_id).castable_to?(current_dtype, casting:)
      raise ConversionError, 'Cannot safely cast Null dataset dtype'
    end
    return HDF5::Empty.new(current_dtype)
  end
  if dtype && Native.datatype_class(type_id) == :H5T_STRING
    raise ConversionError,
          'dtype is not supported for string datasets'
  end
  return read_string(type_id, selection:) if Native.datatype_class(type_id) == :H5T_STRING

  source_dtype = DType.for_hdf5(type_id)
  current_dtype = dtype ? DType.for_symbol(dtype) : source_dtype
  raise ConversionError, "Cannot safely cast #{source_dtype.to_sym} to #{current_dtype.to_sym}" unless
    source_dtype.castable_to?(current_dtype, casting:)

  normalized_selection = Selection.normalize(selection, current_shape)
  return current_dtype.numo_class.zeros(*normalized_selection.result_shape) if normalized_selection.size.zero?
  if current_dtype.kind == :complex && source_dtype.kind != :complex
    raise ConversionError, 'Reading non-complex data as complex requires an explicit Numo cast'
  end

  file_space_id = HDF5::FFI.H5Dget_space(@dataset_id)
  raise NativeError, 'Failed to get dataset dataspace' if file_space_id < 0

  select_hyperslab(file_space_id, normalized_selection)
  memory_space_id = create_memory_dataspace(normalized_selection.result_shape)
  raise NativeError, 'Failed to create memory dataspace' if memory_space_id < 0

  validate_selection_sizes(file_space_id, memory_space_id)

  bytesize = normalized_selection.size * current_dtype.itemsize
  buffer = ::FFI::MemoryPointer.new(:char, bytesize)
  status = HDF5::FFI.H5Dread(@dataset_id, current_dtype.memory_type_id, memory_space_id, file_space_id,
                             HDF5::DEFAULT_PROPERTY_LIST, buffer)
  raise NativeError, 'Failed to read dataset' if status < 0

  result = HDF5::DataHelpers.from_binary(current_dtype, buffer.read_bytes(bytesize),
                                         normalized_selection.result_shape)
  return result unless normalized_selection.scalar?

  scalar = result.extract
  current_dtype.kind == :bool ? !scalar.zero? : scalar
ensure
  Native.close([:H5Tclose, type_id], [:H5Sclose, memory_space_id], [:H5Sclose, file_space_id])
end

#read_array(selection: nil, flatten: false, dtype: nil, casting: :safe) ⇒ Object

# File 'lib/hdf5/dataset.rb', line 500

def read_array(selection: nil, flatten: false, dtype: nil, casting: :safe)
  value = read(selection:, dtype:, casting:)
  return value unless value.is_a?(Numo::NArray)

  array = value.to_a
  flatten ? array.flatten : array
end

#read_into(destination, selection: nil, casting: :safe) ⇒ Object

Raises:

• (ConversionError)

# File 'lib/hdf5/dataset.rb', line 516

def read_into(destination, selection: nil, casting: :safe)
  ensure_open!
  DataHelpers.validate_casting!(casting)
  raise ConversionError, 'read_into destination must be a Numo::NArray' unless destination.is_a?(Numo::NArray)

  current_shape = shape
  raise ShapeError, 'Cannot read a Null dataset into an array' if current_shape.nil?

  expected_shape = Selection.normalize(selection, current_shape).result_shape
  unless destination.shape == expected_shape
    raise ShapeError,
          'read_into destination shape must match selection shape'
  end

  values = read(selection:, dtype: DType.for_numo(destination).to_sym, casting:)
  values = values ? 1 : 0 if expected_shape.empty? && destination.is_a?(Numo::Bit)
  destination.store(values)
end

#resize(new_shape) ⇒ Object

Raises:

• (ShapeError)

# File 'lib/hdf5/dataset.rb', line 379

def resize(new_shape)
  ensure_open!
  raise ShapeError, 'Cannot resize a Null dataset' if shape.nil?
  unless new_shape.is_a?(Array) && new_shape.length == shape.length
    raise ShapeError,
          'Dataset shape must be an Array matching dataset rank'
  end
  unless new_shape.all? { |dimension| dimension.is_a?(Integer) && dimension >= 0 }
    raise ShapeError, 'Dataset dimensions must be non-negative integers'
  end

  maxshape.zip(new_shape).each do |maximum, dimension|
    raise ShapeError, 'Dataset shape exceeds maxshape' if maximum && dimension > maximum
  end

  dimensions = ::FFI::MemoryPointer.new(:ulong_long, new_shape.length)
  dimensions.write_array_of_ulong_long(new_shape)
  status = HDF5::FFI.H5Dset_extent(@dataset_id, dimensions)
  raise NativeError, 'Failed to resize dataset' if status < 0

  self
end

#shape ⇒ Object

# File 'lib/hdf5/dataset.rb', line 298

def shape
  ensure_open!
  dataspace_id = HDF5::FFI.H5Dget_space(@dataset_id)
  raise NativeError, 'Failed to get dataspace' if dataspace_id < 0
  return nil if Native.extent_type(dataspace_id) == :H5S_NULL

  ndims = HDF5::FFI.H5Sget_simple_extent_ndims(dataspace_id)
  raise NativeError, 'Failed to get number of dimensions' if ndims < 0

  dims = ::FFI::MemoryPointer.new(:ulong_long, ndims)
  Native.check(HDF5::FFI.H5Sget_simple_extent_dims(dataspace_id, dims, nil), 'Failed to get dataset shape')

  dims.read_array_of_uint64(ndims)
ensure
  Native.close([:H5Sclose, dataspace_id])
end

#size ⇒ Object

# File 'lib/hdf5/dataset.rb', line 319

def size
  shape&.inject(1, :*) || 0
end

#write(data, selection: nil, casting: :safe) ⇒ Object

Raises:

• (ShapeError)

# File 'lib/hdf5/dataset.rb', line 247

def write(data, selection: nil, casting: :safe)
  ensure_open!
  DataHelpers.validate_casting!(casting)

  return write_string(data, selection:) if dtype.kind == :string || HDF5::StringCodec.string_data?(data)

  current_shape = shape
  raise ShapeError, 'Cannot write to a Null dataset' if current_shape.nil?

  normalized_selection = Selection.normalize(selection, current_shape)
  target_dtype = dtype
  raise ConversionError, 'String datasets require string data' if target_dtype.kind == :string

  values = HDF5::DataHelpers.normalize_data(data, label: 'Dataset data', dtype: target_dtype, casting:, convert: false)
  if HDF5::DataHelpers.scalar?(data) && !normalized_selection.scalar?
    write_scalar(values, target_dtype, normalized_selection) unless normalized_selection.size.zero?
    return data
  end
  raise ShapeError, 'Dataset shape must match data shape' unless values.shape == normalized_selection.result_shape

  return data if normalized_selection.size.zero?

  write_numeric_buffer(HDF5::DataHelpers.buffer_for(values), DType.for_numo(values), normalized_selection)
  data
end