WIP
jdl
2 years ago
parent
e726588b83
commit
2a49c25514
@ -0,0 +1,60 @@
|
||||
package mmtable
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"reflect"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
type bufferedSlice struct {
|
||||
SchemaCol
|
||||
Len func() int
|
||||
SetLen func(int)
|
||||
}
|
||||
|
||||
func newBufferedSlice(
|
||||
buf []byte,
|
||||
sc SchemaCol,
|
||||
targetPtr interface{}, // Pointer to slice.
|
||||
capacity int,
|
||||
) (
|
||||
bs bufferedSlice,
|
||||
sizeInBytes int,
|
||||
err error,
|
||||
) {
|
||||
|
||||
t := elemTypeType(sc.ElemType, sc.Shape)
|
||||
|
||||
bs = bufferedSlice{
|
||||
SchemaCol: sc,
|
||||
}
|
||||
|
||||
// Check for sufficient data size remaining in the buffer.
|
||||
sizeInBytes = capacity * int(t.Size())
|
||||
if len(buf) < sizeInBytes {
|
||||
return bs, 0, ErrDataTruncated
|
||||
}
|
||||
|
||||
v := reflect.ValueOf(targetPtr)
|
||||
|
||||
if reflect.Indirect(v).Type().Elem() != t {
|
||||
log.Print(reflect.Indirect(v).Type().Elem(), t)
|
||||
return bs, 0, fmt.Errorf("%s: %w", sc.Name, ErrIncorrectColType)
|
||||
}
|
||||
|
||||
hdr := (*reflect.SliceHeader)(unsafe.Pointer(v.Pointer()))
|
||||
|
||||
hdr.Data = uintptr(unsafe.Pointer(&buf[0]))
|
||||
hdr.Cap = capacity
|
||||
|
||||
bs.Len = func() int {
|
||||
return hdr.Len
|
||||
}
|
||||
|
||||
bs.SetLen = func(l int) {
|
||||
hdr.Len = l
|
||||
}
|
||||
|
||||
return bs, sizeInBytes, nil
|
||||
}
|
||||
@ -0,0 +1,131 @@
|
||||
package mmtable
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"math/rand"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBufferedSlice_ErrDataTruncated(t *testing.T) {
|
||||
v := [][3]float32{}
|
||||
et, shape := elemTypeForSlice(&v)
|
||||
capacity := 100
|
||||
buf := make([]byte, backingDataSize(et, shape, capacity)-1)
|
||||
sc := SchemaCol{
|
||||
ElemType: et,
|
||||
Shape: shape,
|
||||
Name: "test",
|
||||
}
|
||||
|
||||
_, _, err := newBufferedSlice(buf, sc, &v, capacity)
|
||||
if !errors.Is(err, ErrDataTruncated) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBufferedSlice_ErrIncorrectColType(t *testing.T) {
|
||||
v := [][3]float32{}
|
||||
et, shape := elemTypeForSlice(&v)
|
||||
capacity := 100
|
||||
buf := make([]byte, backingDataSize(et, shape, capacity))
|
||||
sc := SchemaCol{
|
||||
ElemType: et,
|
||||
Shape: shape,
|
||||
Name: "test",
|
||||
}
|
||||
|
||||
v2 := []float32{}
|
||||
_, _, err := newBufferedSlice(buf, sc, &v2, capacity)
|
||||
if !errors.Is(err, ErrIncorrectColType) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBufferedSlice_Simple_Byte(t *testing.T) {
|
||||
v := []float32{}
|
||||
et, shape := elemTypeForSlice(&v)
|
||||
|
||||
capacity := 100
|
||||
buf := make([]byte, backingDataSize(et, shape, capacity))
|
||||
sc := SchemaCol{
|
||||
ElemType: et,
|
||||
Shape: shape,
|
||||
Name: "test",
|
||||
}
|
||||
|
||||
_, n, err := newBufferedSlice(buf, sc, &v, capacity)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != len(buf) {
|
||||
t.Fatal(n, len(buf))
|
||||
}
|
||||
|
||||
// Modify v.
|
||||
v = v[:cap(v)]
|
||||
for i := 0; i < len(v); i++ {
|
||||
v[i] = float32(i) * 1.8
|
||||
}
|
||||
|
||||
// Create new buffered slice on buf.
|
||||
v2 := []float32{}
|
||||
_, n, err = newBufferedSlice(buf, sc, &v2, capacity)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != len(buf) {
|
||||
t.Fatal(n, len(buf))
|
||||
}
|
||||
|
||||
v2 = v2[:len(v)]
|
||||
|
||||
// Ensure it matches v.
|
||||
for i := range v {
|
||||
if v[i] != v2[i] || v[i] != float32(i)*1.8 {
|
||||
t.Fatal(v[i], v2[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBufferedSlice_Length(t *testing.T) {
|
||||
v := []float32{}
|
||||
et, shape := elemTypeForSlice(&v)
|
||||
|
||||
capacity := 100
|
||||
buf := make([]byte, backingDataSize(et, shape, capacity))
|
||||
sc := SchemaCol{
|
||||
ElemType: et,
|
||||
Shape: shape,
|
||||
Name: "test",
|
||||
}
|
||||
|
||||
bs, n, err := newBufferedSlice(buf, sc, &v, capacity)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != len(buf) {
|
||||
t.Fatal(n, len(buf))
|
||||
}
|
||||
|
||||
for i := 0; i < 100; i++ {
|
||||
|
||||
length := 0
|
||||
|
||||
v = v[:length]
|
||||
if len(v) != length || bs.Len() != length {
|
||||
t.Fatal(length, len(v), bs.Len())
|
||||
}
|
||||
|
||||
length = rand.Intn(capacity)
|
||||
v = v[:length]
|
||||
if len(v) != length || bs.Len() != length {
|
||||
t.Fatal(length, len(v), bs.Len())
|
||||
}
|
||||
|
||||
length = rand.Intn(capacity)
|
||||
bs.SetLen(length)
|
||||
if len(v) != length || bs.Len() != length {
|
||||
t.Fatal(length, len(v), bs.Len())
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,76 @@
|
||||
package mmtable
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
)
|
||||
|
||||
type bufferedTable []bufferedSlice
|
||||
|
||||
func newBufferedTable(
|
||||
buf []byte,
|
||||
schema Schema,
|
||||
tblPtr interface{}, // Pointer to table object.
|
||||
capacity int,
|
||||
) (
|
||||
table bufferedTable,
|
||||
err error,
|
||||
) {
|
||||
table = make([]bufferedSlice, 0, len(schema))
|
||||
|
||||
// Map of schema cols by name.
|
||||
scMap := make(map[string]SchemaCol, len(schema))
|
||||
for _, sc := range schema {
|
||||
scMap[sc.Name] = sc
|
||||
}
|
||||
|
||||
// Get pointer to each column by name.
|
||||
targetPtrs := map[string]interface{}{}
|
||||
|
||||
v := reflect.Indirect(reflect.ValueOf(tblPtr))
|
||||
t := reflect.TypeOf(tblPtr).Elem()
|
||||
for i := 0; i < t.NumField(); i++ {
|
||||
field := t.Field(i)
|
||||
name := field.Tag.Get("mmtable")
|
||||
if name == "-" {
|
||||
continue
|
||||
}
|
||||
if name == "" {
|
||||
name = field.Name
|
||||
}
|
||||
if _, ok := scMap[name]; !ok {
|
||||
return table, fmt.Errorf("Unknown column %s: %w", name, ErrUnknownColumn)
|
||||
}
|
||||
|
||||
targetPtrs[name] = v.Field(i).Addr().Interface()
|
||||
}
|
||||
|
||||
for _, sc := range schema {
|
||||
target := targetPtrs[sc.Name]
|
||||
if target == nil {
|
||||
continue
|
||||
}
|
||||
bs, n, err := newBufferedSlice(buf, sc, target, capacity)
|
||||
if err != nil {
|
||||
return table, err
|
||||
}
|
||||
table = append(table, bs)
|
||||
buf = buf[n:]
|
||||
}
|
||||
|
||||
return table, nil
|
||||
}
|
||||
|
||||
func (bbt bufferedTable) Lengths() []int {
|
||||
lengths := make([]int, len(bbt))
|
||||
for i, ci := range bbt {
|
||||
lengths[i] = ci.Len()
|
||||
}
|
||||
return lengths
|
||||
}
|
||||
|
||||
func (bbt bufferedTable) SetLengths(lengths []int) {
|
||||
for i, ci := range bbt {
|
||||
ci.SetLen(lengths[i])
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,233 @@
|
||||
package mmtable
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"math/rand"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type TestTable struct {
|
||||
Byte []byte
|
||||
Bool []bool
|
||||
Int8 []int8
|
||||
Int16 []int16
|
||||
Int32 []int32
|
||||
Int64 []int64
|
||||
Uint8 []uint8
|
||||
Uint16 []uint16
|
||||
Uint32 []uint32
|
||||
Uint64 []uint64
|
||||
Float32 []float32
|
||||
Float64 []float64
|
||||
Hidden []int64 `mmtable:"-"`
|
||||
}
|
||||
|
||||
var testSchema = Schema([]SchemaCol{
|
||||
{Name: "Byte", ElemType: ElemTypeByte},
|
||||
{Name: "Bool", ElemType: ElemTypeBool},
|
||||
{Name: "Int8", ElemType: ElemTypeInt8},
|
||||
{Name: "Int16", ElemType: ElemTypeInt16},
|
||||
{Name: "Int32", ElemType: ElemTypeInt32},
|
||||
{Name: "Int64", ElemType: ElemTypeInt64},
|
||||
{Name: "Uint8", ElemType: ElemTypeUint8},
|
||||
{Name: "Uint16", ElemType: ElemTypeUint16},
|
||||
{Name: "Uint32", ElemType: ElemTypeUint32},
|
||||
{Name: "Uint64", ElemType: ElemTypeUint64},
|
||||
{Name: "Float32", ElemType: ElemTypeFloat32},
|
||||
{Name: "Float64", ElemType: ElemTypeFloat64},
|
||||
})
|
||||
|
||||
func TestBufferedTable(t *testing.T) {
|
||||
capacity := 100
|
||||
buf := make([]byte, testSchema.dataSize(capacity))
|
||||
tt := &TestTable{}
|
||||
|
||||
bt, err := newBufferedTable(buf, testSchema, tt, capacity)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Write random data to each column.
|
||||
for i := 0; i < rand.Intn(capacity); i++ {
|
||||
tt.Byte = append(tt.Byte, byte(rand.Int()))
|
||||
}
|
||||
for i := 0; i < rand.Intn(capacity); i++ {
|
||||
tt.Bool = append(tt.Bool, rand.Int()%2 == 0)
|
||||
}
|
||||
for i := 0; i < rand.Intn(capacity); i++ {
|
||||
tt.Int8 = append(tt.Int8, int8(rand.Int()))
|
||||
}
|
||||
for i := 0; i < rand.Intn(capacity); i++ {
|
||||
tt.Int16 = append(tt.Int16, int16(rand.Int()))
|
||||
}
|
||||
for i := 0; i < rand.Intn(capacity); i++ {
|
||||
tt.Int32 = append(tt.Int32, int32(rand.Int()))
|
||||
}
|
||||
for i := 0; i < rand.Intn(capacity); i++ {
|
||||
tt.Int64 = append(tt.Int64, int64(rand.Int()))
|
||||
}
|
||||
for i := 0; i < rand.Intn(capacity); i++ {
|
||||
tt.Uint8 = append(tt.Uint8, uint8(rand.Int()))
|
||||
}
|
||||
for i := 0; i < rand.Intn(capacity); i++ {
|
||||
tt.Uint16 = append(tt.Uint16, uint16(rand.Int()))
|
||||
}
|
||||
for i := 0; i < rand.Intn(capacity); i++ {
|
||||
tt.Uint32 = append(tt.Uint32, uint32(rand.Int()))
|
||||
}
|
||||
for i := 0; i < rand.Intn(capacity); i++ {
|
||||
tt.Uint64 = append(tt.Uint64, uint64(rand.Int()))
|
||||
}
|
||||
for i := 0; i < rand.Intn(capacity); i++ {
|
||||
tt.Float32 = append(tt.Float32, rand.Float32())
|
||||
}
|
||||
for i := 0; i < rand.Intn(capacity); i++ {
|
||||
tt.Float64 = append(tt.Float64, rand.Float64())
|
||||
}
|
||||
|
||||
// Check lengths.
|
||||
lengths := []int{
|
||||
len(tt.Byte),
|
||||
len(tt.Bool),
|
||||
len(tt.Int8),
|
||||
len(tt.Int16),
|
||||
len(tt.Int32),
|
||||
len(tt.Int64),
|
||||
len(tt.Uint8),
|
||||
len(tt.Uint16),
|
||||
len(tt.Uint32),
|
||||
len(tt.Uint64),
|
||||
len(tt.Float32),
|
||||
len(tt.Float64),
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(bt.Lengths(), lengths) {
|
||||
t.Fatal(lengths, bt.Lengths())
|
||||
}
|
||||
|
||||
// Set and check lengths.
|
||||
for i := range lengths {
|
||||
lengths[i] = rand.Intn(capacity)
|
||||
}
|
||||
|
||||
bt.SetLengths(lengths)
|
||||
|
||||
lengths = []int{
|
||||
len(tt.Byte),
|
||||
len(tt.Bool),
|
||||
len(tt.Int8),
|
||||
len(tt.Int16),
|
||||
len(tt.Int32),
|
||||
len(tt.Int64),
|
||||
len(tt.Uint8),
|
||||
len(tt.Uint16),
|
||||
len(tt.Uint32),
|
||||
len(tt.Uint64),
|
||||
len(tt.Float32),
|
||||
len(tt.Float64),
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(bt.Lengths(), lengths) {
|
||||
t.Fatal(lengths, bt.Lengths())
|
||||
}
|
||||
|
||||
tt2 := &TestTable{}
|
||||
bt2, err := newBufferedTable(buf, testSchema, tt2, capacity)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
bt2.SetLengths(bt.Lengths())
|
||||
|
||||
if !reflect.DeepEqual(tt, tt2) {
|
||||
t.Fatal(tt, tt2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewBufferedTable_ErrUnknownColumn(t *testing.T) {
|
||||
type MyTable struct {
|
||||
M [3][3]float32
|
||||
X [3]float32 `mmtable:"x"`
|
||||
}
|
||||
|
||||
schema := Schema{
|
||||
{
|
||||
Name: "M",
|
||||
ElemType: ElemTypeFloat32,
|
||||
Shape: Shape(3, 3),
|
||||
}, {
|
||||
Name: "X",
|
||||
ElemType: ElemTypeFloat32,
|
||||
Shape: Shape(3),
|
||||
},
|
||||
}
|
||||
|
||||
capacity := 100
|
||||
buf := make([]byte, schema.dataSize(capacity))
|
||||
tt := &MyTable{}
|
||||
|
||||
_, err := newBufferedTable(buf, schema, tt, capacity)
|
||||
if !errors.Is(err, ErrUnknownColumn) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewBufferedTable_MissingTarget(t *testing.T) {
|
||||
type MyTable struct {
|
||||
M [][3][3]byte
|
||||
X [][3]float32
|
||||
}
|
||||
|
||||
schema := Schema{
|
||||
{
|
||||
Name: "M",
|
||||
ElemType: ElemTypeByte,
|
||||
Shape: Shape(3, 3),
|
||||
}, {
|
||||
Name: "X",
|
||||
ElemType: ElemTypeFloat32,
|
||||
Shape: Shape(3),
|
||||
}, {
|
||||
Name: "Z",
|
||||
ElemType: ElemTypeFloat32,
|
||||
},
|
||||
}
|
||||
|
||||
capacity := 100
|
||||
buf := make([]byte, schema.dataSize(capacity))
|
||||
tt := &MyTable{}
|
||||
|
||||
_, err := newBufferedTable(buf, schema, tt, capacity)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewBufferedTable_SchemaMismatch(t *testing.T) {
|
||||
type MyTable struct {
|
||||
M [][3][3]byte
|
||||
X []float32
|
||||
}
|
||||
|
||||
schema := Schema{
|
||||
{
|
||||
Name: "M",
|
||||
ElemType: ElemTypeByte,
|
||||
Shape: Shape(3, 3),
|
||||
}, {
|
||||
Name: "X",
|
||||
ElemType: ElemTypeFloat32,
|
||||
Shape: Shape(3),
|
||||
},
|
||||
}
|
||||
|
||||
capacity := 100
|
||||
buf := make([]byte, schema.dataSize(capacity))
|
||||
tt := &MyTable{}
|
||||
|
||||
_, err := newBufferedTable(buf, schema, tt, capacity)
|
||||
if !errors.Is(err, ErrIncorrectColType) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,7 @@
|
||||
package mmtable
|
||||
|
||||
const (
|
||||
Version = 1
|
||||
columnMetaSize = 64
|
||||
ColumnNameMaxLength = 64
|
||||
)
|
||||
@ -0,0 +1,40 @@
|
||||
package mmtable
|
||||
|
||||
import "reflect"
|
||||
|
||||
type ElemType byte
|
||||
|
||||
const (
|
||||
ElemTypeByte ElemType = 0
|
||||
ElemTypeBool ElemType = 1
|
||||
ElemTypeInt8 ElemType = 2
|
||||
ElemTypeInt16 ElemType = 3
|
||||
ElemTypeInt32 ElemType = 4
|
||||
ElemTypeInt64 ElemType = 5
|
||||
ElemTypeUint8 ElemType = 6
|
||||
ElemTypeUint16 ElemType = 7
|
||||
ElemTypeUint32 ElemType = 8
|
||||
ElemTypeUint64 ElemType = 9
|
||||
ElemTypeFloat32 ElemType = 10
|
||||
ElemTypeFloat64 ElemType = 11
|
||||
elemTypeMax ElemType = 12
|
||||
)
|
||||
|
||||
func elemTypeIsValid(t ElemType) bool {
|
||||
return t < elemTypeMax
|
||||
}
|
||||
|
||||
var nativeElemType = []reflect.Type{
|
||||
reflect.TypeOf(byte(0)),
|
||||
reflect.TypeOf(false),
|
||||
reflect.TypeOf(int8(0)),
|
||||
reflect.TypeOf(int16(0)),
|
||||
reflect.TypeOf(int32(0)),
|
||||
reflect.TypeOf(int64(0)),
|
||||
reflect.TypeOf(uint8(0)),
|
||||
reflect.TypeOf(uint16(0)),
|
||||
reflect.TypeOf(uint32(0)),
|
||||
reflect.TypeOf(uint64(0)),
|
||||
reflect.TypeOf(float32(0)),
|
||||
reflect.TypeOf(float64(0)),
|
||||
}
|
||||
@ -0,0 +1,16 @@
|
||||
package mmtable
|
||||
|
||||
import "errors"
|
||||
|
||||
var (
|
||||
ErrUnknownElementType = errors.New("unknown element type")
|
||||
ErrZeroArraySize = errors.New("zero array size")
|
||||
ErrNameTooLong = errors.New("name is too long")
|
||||
ErrNameEmpty = errors.New("name is empty")
|
||||
ErrNameDuplicate = errors.New("duplicate name")
|
||||
|
||||
ErrCorruptSchema = errors.New("corrupt schema data")
|
||||
ErrDataTruncated = errors.New("truncated data")
|
||||
ErrIncorrectColType = errors.New("incorrect column type")
|
||||
ErrUnknownColumn = errors.New("unknown column")
|
||||
)
|
||||
@ -0,0 +1,127 @@
|
||||
package mmtable
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
func Shape(in ...uint8) [3]uint8 {
|
||||
out := [3]uint8{}
|
||||
if len(in) > 3 {
|
||||
in = in[:3]
|
||||
}
|
||||
for i := range in {
|
||||
if in[i] == 0 {
|
||||
break
|
||||
}
|
||||
out[i] = in[i]
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type SchemaCol struct {
|
||||
ElemType ElemType
|
||||
Shape [3]uint8 // Shape of each element. Zero value => scalar.
|
||||
Name string // Max length is 64 bytes.
|
||||
}
|
||||
|
||||
func (sc SchemaCol) validate() error {
|
||||
if !elemTypeIsValid(sc.ElemType) {
|
||||
return fmt.Errorf("%d: %w", sc.ElemType, ErrUnknownElementType)
|
||||
}
|
||||
|
||||
if len(sc.Name) > ColumnNameMaxLength {
|
||||
return fmt.Errorf("%s: %w", sc.Name, ErrNameTooLong)
|
||||
}
|
||||
|
||||
if len(sc.Name) == 0 {
|
||||
return fmt.Errorf("%s: %w", sc.Name, ErrNameEmpty)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type Schema []SchemaCol
|
||||
|
||||
func (s Schema) Validate() error {
|
||||
names := map[string]bool{}
|
||||
for i := range s {
|
||||
if _, ok := names[s[i].Name]; ok {
|
||||
return fmt.Errorf("%s: %w", s[i].Name, ErrNameDuplicate)
|
||||
}
|
||||
names[s[i].Name] = true
|
||||
if err := s[i].validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s Schema) schemaSize() int {
|
||||
return 2 + (columnMetaSize+ColumnNameMaxLength)*len(s)
|
||||
}
|
||||
|
||||
func (s Schema) dataSize(capacity int) (size int) {
|
||||
for _, sc := range s {
|
||||
t := elemTypeType(sc.ElemType, sc.Shape)
|
||||
size += int(t.Size())
|
||||
}
|
||||
return size * capacity
|
||||
}
|
||||
|
||||
// out should have the capacity of at least `s.serializedSize()`.
|
||||
func (s Schema) write(out []byte) {
|
||||
|
||||
// Write the number of columns.
|
||||
binary.LittleEndian.PutUint16(out[:2], uint16(len(s)))
|
||||
out = out[2:]
|
||||
|
||||
for _, sc := range s {
|
||||
out[0] = byte(sc.ElemType)
|
||||
out[1] = sc.Shape[0]
|
||||
out[2] = sc.Shape[1]
|
||||
out[3] = sc.Shape[2]
|
||||
out = out[columnMetaSize:]
|
||||
|
||||
for i, b := range []byte(sc.Name) {
|
||||
out[i] = b
|
||||
}
|
||||
|
||||
for i := len(sc.Name); i < ColumnNameMaxLength; i++ {
|
||||
out[i] = 0
|
||||
}
|
||||
|
||||
out = out[ColumnNameMaxLength:]
|
||||
}
|
||||
}
|
||||
|
||||
func readSchema(buf []byte) (Schema, error) {
|
||||
colCount := binary.LittleEndian.Uint16(buf[:2])
|
||||
|
||||
schema := Schema(make([]SchemaCol, colCount))
|
||||
|
||||
if len(buf) < schema.schemaSize() {
|
||||
return schema, ErrCorruptSchema
|
||||
}
|
||||
|
||||
buf = buf[2:]
|
||||
|
||||
for i := range schema {
|
||||
schema[i].ElemType = ElemType(buf[0])
|
||||
schema[i].Shape[0] = buf[1]
|
||||
schema[i].Shape[1] = buf[2]
|
||||
schema[i].Shape[2] = buf[3]
|
||||
buf = buf[columnMetaSize:]
|
||||
|
||||
length := 0
|
||||
for ; length < ColumnNameMaxLength; length++ {
|
||||
if buf[length] == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
schema[i].Name = string(buf[:length])
|
||||
buf = buf[ColumnNameMaxLength:]
|
||||
}
|
||||
|
||||
return schema, schema.Validate()
|
||||
}
|
||||
@ -0,0 +1,225 @@
|
||||
package mmtable
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestShape(t *testing.T) {
|
||||
type TestCase struct {
|
||||
in []uint8
|
||||
out [3]uint8
|
||||
}
|
||||
|
||||
cases := []TestCase{
|
||||
{in: []uint8{}, out: [3]uint8{0, 0, 0}},
|
||||
{in: []uint8{0}, out: [3]uint8{0, 0, 0}},
|
||||
{in: []uint8{0, 0}, out: [3]uint8{0, 0, 0}},
|
||||
{in: []uint8{0, 0, 0}, out: [3]uint8{0, 0, 0}},
|
||||
{in: []uint8{0, 0, 0, 0}, out: [3]uint8{0, 0, 0}},
|
||||
{in: []uint8{1}, out: [3]uint8{1, 0, 0}},
|
||||
{in: []uint8{0, 1}, out: [3]uint8{0, 0, 0}},
|
||||
{in: []uint8{1, 2, 3}, out: [3]uint8{1, 2, 3}},
|
||||
{in: []uint8{1, 0, 1}, out: [3]uint8{1, 0, 0}},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
out := Shape(tc.in...)
|
||||
if !reflect.DeepEqual(out, tc.out) {
|
||||
t.Fatal(out, tc.out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSchemaCol_validate(t *testing.T) {
|
||||
type TestCase struct {
|
||||
sc SchemaCol
|
||||
err error
|
||||
}
|
||||
|
||||
cases := []TestCase{
|
||||
{
|
||||
sc: SchemaCol{ElemType: elemTypeMax, Shape: Shape(), Name: "x"},
|
||||
err: ErrUnknownElementType,
|
||||
}, {
|
||||
sc: SchemaCol{
|
||||
ElemType: ElemTypeBool,
|
||||
Shape: Shape(),
|
||||
Name: strings.Repeat("x", ColumnNameMaxLength) + "z",
|
||||
},
|
||||
err: ErrNameTooLong,
|
||||
}, {
|
||||
sc: SchemaCol{ElemType: ElemTypeInt8, Shape: Shape(), Name: ""},
|
||||
err: ErrNameEmpty,
|
||||
}, {
|
||||
sc: SchemaCol{ElemType: ElemTypeInt8, Shape: Shape(), Name: "x"},
|
||||
err: nil,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
err := tc.sc.validate()
|
||||
if !errors.Is(err, tc.err) {
|
||||
t.Fatal(tc.sc, tc.err, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSchema_Validate(t *testing.T) {
|
||||
schema := Schema([]SchemaCol{
|
||||
{
|
||||
ElemType: ElemTypeInt64,
|
||||
Shape: Shape(2, 2, 0),
|
||||
Name: "col1",
|
||||
},
|
||||
})
|
||||
|
||||
if err := schema.Validate(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
schema = append(schema, SchemaCol{
|
||||
ElemType: elemTypeMax,
|
||||
Shape: Shape(),
|
||||
Name: "col2",
|
||||
})
|
||||
|
||||
if err := schema.Validate(); !errors.Is(err, ErrUnknownElementType) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
schema[1].ElemType = ElemTypeUint32
|
||||
schema[1].Name = "col1"
|
||||
|
||||
if err := schema.Validate(); !errors.Is(err, ErrNameDuplicate) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSchema_schemaSize(t *testing.T) {
|
||||
schema := Schema([]SchemaCol{
|
||||
{
|
||||
ElemType: ElemTypeByte,
|
||||
Shape: Shape(2, 0, 0),
|
||||
Name: "col1",
|
||||
}, {
|
||||
ElemType: ElemTypeInt32,
|
||||
Shape: Shape(2, 2, 2),
|
||||
Name: "col2",
|
||||
}, {
|
||||
ElemType: ElemTypeFloat32,
|
||||
Shape: Shape(),
|
||||
Name: "col3",
|
||||
},
|
||||
})
|
||||
|
||||
expected := 2 + 128*3
|
||||
if schema.schemaSize() != expected {
|
||||
t.Fatal(schema.schemaSize(), expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSchema_dataSize(t *testing.T) {
|
||||
schema := Schema([]SchemaCol{
|
||||
{
|
||||
ElemType: ElemTypeByte,
|
||||
Shape: Shape(2, 0, 0),
|
||||
Name: "col1",
|
||||
}, {
|
||||
ElemType: ElemTypeInt32,
|
||||
Shape: Shape(2, 2, 2),
|
||||
Name: "col2",
|
||||
}, {
|
||||
ElemType: ElemTypeFloat64,
|
||||
Shape: Shape(),
|
||||
Name: "col3",
|
||||
},
|
||||
})
|
||||
|
||||
capacity := 100
|
||||
expected := capacity * (2 + 8*4 + 8)
|
||||
if schema.dataSize(capacity) != expected {
|
||||
t.Fatal(schema.dataSize(capacity), expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSchema_write_readSchema(t *testing.T) {
|
||||
schema := Schema{}
|
||||
|
||||
out := make([]byte, schema.schemaSize())
|
||||
schema.write(out)
|
||||
s2, err := readSchema(out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(schema, s2) {
|
||||
t.Fatal(schema, s2)
|
||||
}
|
||||
|
||||
schema = append(schema, SchemaCol{
|
||||
ElemType: ElemTypeByte,
|
||||
Shape: Shape(2, 0, 0),
|
||||
Name: "col1",
|
||||
})
|
||||
|
||||
out = make([]byte, schema.schemaSize())
|
||||
schema.write(out)
|
||||
s2, err = readSchema(out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(schema, s2) {
|
||||
t.Fatal(schema, s2)
|
||||
}
|
||||
|
||||
schema = append(schema, SchemaCol{
|
||||
ElemType: ElemTypeFloat64,
|
||||
Shape: Shape(2, 2, 3),
|
||||
Name: "col2",
|
||||
})
|
||||
|
||||
out = make([]byte, schema.schemaSize())
|
||||
schema.write(out)
|
||||
s2, err = readSchema(out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(schema, s2) {
|
||||
t.Fatal(schema, s2)
|
||||
}
|
||||
|
||||
schema = append(schema, SchemaCol{
|
||||
ElemType: ElemTypeUint16,
|
||||
Shape: Shape(2, 0, 0),
|
||||
Name: "col3",
|
||||
})
|
||||
|
||||
out = make([]byte, schema.schemaSize())
|
||||
schema.write(out)
|
||||
s2, err = readSchema(out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(schema, s2) {
|
||||
t.Fatal(schema, s2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadSchema_ErrCorruptSchema(t *testing.T) {
|
||||
schema := Schema([]SchemaCol{
|
||||
{
|
||||
ElemType: ElemTypeByte,
|
||||
Shape: Shape(2, 0, 0),
|
||||
Name: "col1",
|
||||
},
|
||||
})
|
||||
|
||||
out := make([]byte, schema.schemaSize())
|
||||
schema.write(out)
|
||||
out = out[:len(out)-1]
|
||||
if _, err := readSchema(out); !errors.Is(err, ErrCorruptSchema) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,28 @@
|
||||
package mmtable
|
||||
|
||||
import "os"
|
||||
|
||||
type Table struct {
|
||||
dir string // Directory where the table lives.
|
||||
lockPath string // Path to lock file.
|
||||
dataPath string // Path to data file.
|
||||
lengthPath string // Path to length file.
|
||||
lengthStagingPath string // Length file staging path for atomic writes.
|
||||
lockFile *os.File // Not nil when opened for writing.
|
||||
dataFile *os.File // The data file.
|
||||
mapped []byte // The mmapped file data.
|
||||
writable bool // True if the data file is opened for writing.
|
||||
schema Schema // The schema.
|
||||
cap int // The capacity of the table.
|
||||
table bufferedTable // Col info in schema order.
|
||||
}
|
||||
|
||||
func Create(dir string, schema Schema, cap int64) (*Table, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// TODO: Create
|
||||
// TODO: Open
|
||||
// TODO: OpenRW
|
||||
// TODO: Close
|
||||
// TODO: Sync() // Syncs data and column lengths.
|
||||
@ -0,0 +1,16 @@
|
||||
package mmtable
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
)
|
||||
|
||||
func elemTypeType(el ElemType, shape [3]uint8) reflect.Type {
|
||||
dType := nativeElemType[el]
|
||||
for _, dim := range shape {
|
||||
if dim == 0 {
|
||||
break
|
||||
}
|
||||
dType = reflect.ArrayOf(int(dim), dType)
|
||||
}
|
||||
return dType
|
||||
}
|
||||
@ -0,0 +1,37 @@
|
||||
package mmtable
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
)
|
||||
|
||||
func backingDataSize(et ElemType, shape [3]uint8, capacity int) int {
|
||||
t := elemTypeType(et, shape)
|
||||
return int(t.Size()) * capacity
|
||||
}
|
||||
|
||||
func elemTypeForSlice(x interface{}) (ElemType, [3]uint8) {
|
||||
// Value of the interface.
|
||||
v := reflect.ValueOf(x)
|
||||
|
||||
// Type of element.
|
||||
t := reflect.Indirect(v).Type().Elem()
|
||||
shape := [3]uint8{}
|
||||
|
||||
i := 0
|
||||
for t.Kind() == reflect.Array {
|
||||
shape[i] = uint8(t.Len())
|
||||
i++
|
||||
t = t.Elem()
|
||||
}
|
||||
|
||||
for et, tt := range nativeElemType {
|
||||
if tt == t {
|
||||
return ElemType(et), shape
|
||||
}
|
||||
}
|
||||
|
||||
panic(fmt.Sprintf("Unknown type: %v", t))
|
||||
|
||||
return 255, shape
|
||||
}
|
||||
Loading…
Reference in New Issue