Module: HDF5::StringCodec

Defined in:
lib/hdf5/string_codec.rb

Constant Summary collapse

UTF8 = 
1
VARIABLE_SIZE = 
(1 << (::FFI.type_size(:size_t) * 8)) - 1

Class Method Summary collapse

Class Method Details

.array_shape(value) ⇒ Object 



76
77
78
79
 
# File 'lib/hdf5/string_codec.rb', line 76

def array_shape(value)
  return [] unless value.is_a?(Array)
  DataHelpers.array_shape(value)
end

.buffer_for(value) ⇒ Object 



20
21
22
 
# File 'lib/hdf5/string_codec.rb', line 20

def buffer_for(value)
  buffer_for_values([normalize(value)])
end

.buffer_for_values(values, encoding: Encoding::UTF_8) ⇒ Object 



24
25
26
27
28
29
 
# File 'lib/hdf5/string_codec.rb', line 24

def buffer_for_values(values, encoding: Encoding::UTF_8)
  pointers = values.map { |value| ::FFI::MemoryPointer.from_string(normalize(value, encoding:)) }
  buffer = ::FFI::MemoryPointer.new(:pointer, pointers.length)
  buffer.write_array_of_pointer(pointers)
  [buffer, pointers]
end

.check(status, operation) ⇒ Object 



107
108
109
 
# File 'lib/hdf5/string_codec.rb', line 107

def check(status, operation)
  Native.check(status, "Failed to #{operation}")
end

.datatype_id ⇒ Object 



8
9
10
11
12
13
14
15
16
17
18
 
# File 'lib/hdf5/string_codec.rb', line 8

def datatype_id
  type_id = HDF5::FFI.H5Tcopy(HDF5::FFI.H5T_C_S1)
  raise NativeError, 'Failed to create string datatype' if type_id < 0

  check(HDF5::FFI.H5Tset_size(type_id, VARIABLE_SIZE), 'set variable string size')
  check(HDF5::FFI.H5Tset_cset(type_id, UTF8), 'set UTF-8 string encoding')
  type_id
rescue StandardError
  Native.close([:H5Tclose, type_id])
  raise
end

.encoding_for(type_id) ⇒ Object 



85
86
87
88
89
90
91
92
 
# File 'lib/hdf5/string_codec.rb', line 85

def encoding_for(type_id)
  case HDF5::FFI.H5Tget_cset(type_id)
  when :H5T_CSET_ERROR then raise NativeError, 'Failed to get string encoding'
  when :H5T_CSET_ASCII then Encoding::US_ASCII
  when :H5T_CSET_UTF8 then Encoding::UTF_8
  else raise UnsupportedTypeError, 'Unsupported HDF5 string encoding'
  end
end

.normalize(value, encoding: Encoding::UTF_8) ⇒ Object 



94
95
96
97
98
99
100
101
102
103
104
105
 
# File 'lib/hdf5/string_codec.rb', line 94

def normalize(value, encoding: Encoding::UTF_8)
  raise ConversionError, 'String dataset data must be a String' unless value.is_a?(String)
  raise ConversionError, 'String data has an invalid encoding' unless value.valid_encoding?
  raise ConversionError, 'Variable-length strings cannot contain NUL bytes' if value.include?("\0")

  result = value.encode(encoding)
  raise ConversionError, "String data is not valid #{encoding.name}" unless result.valid_encoding?

  result
rescue EncodingError
  raise ConversionError, "String data must be representable as #{encoding.name}"
end

.normalize_data(value, encoding: Encoding::UTF_8) ⇒ Object 



64
65
66
67
68
69
70
71
72
73
74
 
# File 'lib/hdf5/string_codec.rb', line 64

def normalize_data(value, encoding: Encoding::UTF_8)
  return [[normalize(value, encoding:)], []] if value.is_a?(String)
  return [value.to_a.flatten.map { |item| normalize(item, encoding:) }, value.shape] if value.is_a?(Numo::RObject)

  unless value.is_a?(Array)
    raise ConversionError, 'String data must be a String, Array of strings, or Numo::RObject'
  end

  shape = array_shape(value)
  [value.flatten.map { |item| normalize(item, encoding:) }, shape]
end

.read(buffer, encoding: Encoding::UTF_8) ⇒ Object 



31
32
33
 
# File 'lib/hdf5/string_codec.rb', line 31

def read(buffer, encoding: Encoding::UTF_8)
  read_values(buffer, 1, [], encoding:)
end

.read_values(buffer, count, shape, encoding: Encoding::UTF_8) ⇒ Object 



35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
 
# File 'lib/hdf5/string_codec.rb', line 35

def read_values(buffer, count, shape, encoding: Encoding::UTF_8)
  return Numo::RObject.new(*shape) if count.zero?

  values = buffer.read_array_of_pointer(count).map do |pointer|
    next nil if pointer.null?

    value = pointer.read_string.force_encoding(encoding)
    raise ConversionError, "String data is not valid #{encoding.name}" unless value.valid_encoding?

    value
  end
  return values.first if shape.empty?

  Numo::RObject.cast(values).reshape(*shape)
end

.string_data?(value) ⇒ Boolean

Returns:

  • (Boolean) 


51
52
53
54
55
56
57
58
59
60
61
62
 
# File 'lib/hdf5/string_codec.rb', line 51

def string_data?(value)
  return true if value.is_a?(String)

  values = if value.is_a?(Numo::RObject)
             value.to_a.flatten
           elsif value.is_a?(Array)
             value.flatten
           else
             []
           end
  !values.empty? && values.all? { |item| item.is_a?(String) }
end

.variable?(type_id) ⇒ Boolean

Returns:

  • (Boolean) 


81
82
83
 
# File 'lib/hdf5/string_codec.rb', line 81

def variable?(type_id)
  Native.check(HDF5::FFI.H5Tis_variable_str(type_id), 'Failed to inspect string datatype').positive?
end