diff --git a/bufferedslice.go b/bufferedslice.go new file mode 100644 index 0000000..2d27ef6 --- /dev/null +++ b/bufferedslice.go @@ -0,0 +1,60 @@ +package mmtable + +import ( + "fmt" + "log" + "reflect" + "unsafe" +) + +type bufferedSlice struct { + SchemaCol + Len func() int + SetLen func(int) +} + +func newBufferedSlice( + buf []byte, + sc SchemaCol, + targetPtr interface{}, // Pointer to slice. + capacity int, +) ( + bs bufferedSlice, + sizeInBytes int, + err error, +) { + + t := elemTypeType(sc.ElemType, sc.Shape) + + bs = bufferedSlice{ + SchemaCol: sc, + } + + // Check for sufficient data size remaining in the buffer. + sizeInBytes = capacity * int(t.Size()) + if len(buf) < sizeInBytes { + return bs, 0, ErrDataTruncated + } + + v := reflect.ValueOf(targetPtr) + + if reflect.Indirect(v).Type().Elem() != t { + log.Print(reflect.Indirect(v).Type().Elem(), t) + return bs, 0, fmt.Errorf("%s: %w", sc.Name, ErrIncorrectColType) + } + + hdr := (*reflect.SliceHeader)(unsafe.Pointer(v.Pointer())) + + hdr.Data = uintptr(unsafe.Pointer(&buf[0])) + hdr.Cap = capacity + + bs.Len = func() int { + return hdr.Len + } + + bs.SetLen = func(l int) { + hdr.Len = l + } + + return bs, sizeInBytes, nil +} diff --git a/bufferedslice_test.go b/bufferedslice_test.go new file mode 100644 index 0000000..0d68ead --- /dev/null +++ b/bufferedslice_test.go @@ -0,0 +1,131 @@ +package mmtable + +import ( + "errors" + "math/rand" + "testing" +) + +func TestBufferedSlice_ErrDataTruncated(t *testing.T) { + v := [][3]float32{} + et, shape := elemTypeForSlice(&v) + capacity := 100 + buf := make([]byte, backingDataSize(et, shape, capacity)-1) + sc := SchemaCol{ + ElemType: et, + Shape: shape, + Name: "test", + } + + _, _, err := newBufferedSlice(buf, sc, &v, capacity) + if !errors.Is(err, ErrDataTruncated) { + t.Fatal(err) + } +} + +func TestBufferedSlice_ErrIncorrectColType(t *testing.T) { + v := [][3]float32{} + et, shape := elemTypeForSlice(&v) + capacity := 100 + buf := make([]byte, backingDataSize(et, shape, capacity)) + sc := SchemaCol{ + ElemType: et, + Shape: shape, + Name: "test", + } + + v2 := []float32{} + _, _, err := newBufferedSlice(buf, sc, &v2, capacity) + if !errors.Is(err, ErrIncorrectColType) { + t.Fatal(err) + } +} + +func TestBufferedSlice_Simple_Byte(t *testing.T) { + v := []float32{} + et, shape := elemTypeForSlice(&v) + + capacity := 100 + buf := make([]byte, backingDataSize(et, shape, capacity)) + sc := SchemaCol{ + ElemType: et, + Shape: shape, + Name: "test", + } + + _, n, err := newBufferedSlice(buf, sc, &v, capacity) + if err != nil { + t.Fatal(err) + } + if n != len(buf) { + t.Fatal(n, len(buf)) + } + + // Modify v. + v = v[:cap(v)] + for i := 0; i < len(v); i++ { + v[i] = float32(i) * 1.8 + } + + // Create new buffered slice on buf. + v2 := []float32{} + _, n, err = newBufferedSlice(buf, sc, &v2, capacity) + if err != nil { + t.Fatal(err) + } + if n != len(buf) { + t.Fatal(n, len(buf)) + } + + v2 = v2[:len(v)] + + // Ensure it matches v. + for i := range v { + if v[i] != v2[i] || v[i] != float32(i)*1.8 { + t.Fatal(v[i], v2[i]) + } + } +} + +func TestBufferedSlice_Length(t *testing.T) { + v := []float32{} + et, shape := elemTypeForSlice(&v) + + capacity := 100 + buf := make([]byte, backingDataSize(et, shape, capacity)) + sc := SchemaCol{ + ElemType: et, + Shape: shape, + Name: "test", + } + + bs, n, err := newBufferedSlice(buf, sc, &v, capacity) + if err != nil { + t.Fatal(err) + } + if n != len(buf) { + t.Fatal(n, len(buf)) + } + + for i := 0; i < 100; i++ { + + length := 0 + + v = v[:length] + if len(v) != length || bs.Len() != length { + t.Fatal(length, len(v), bs.Len()) + } + + length = rand.Intn(capacity) + v = v[:length] + if len(v) != length || bs.Len() != length { + t.Fatal(length, len(v), bs.Len()) + } + + length = rand.Intn(capacity) + bs.SetLen(length) + if len(v) != length || bs.Len() != length { + t.Fatal(length, len(v), bs.Len()) + } + } +} diff --git a/bufferedtable.go b/bufferedtable.go new file mode 100644 index 0000000..4aa4d0e --- /dev/null +++ b/bufferedtable.go @@ -0,0 +1,76 @@ +package mmtable + +import ( + "fmt" + "reflect" +) + +type bufferedTable []bufferedSlice + +func newBufferedTable( + buf []byte, + schema Schema, + tblPtr interface{}, // Pointer to table object. + capacity int, +) ( + table bufferedTable, + err error, +) { + table = make([]bufferedSlice, 0, len(schema)) + + // Map of schema cols by name. + scMap := make(map[string]SchemaCol, len(schema)) + for _, sc := range schema { + scMap[sc.Name] = sc + } + + // Get pointer to each column by name. + targetPtrs := map[string]interface{}{} + + v := reflect.Indirect(reflect.ValueOf(tblPtr)) + t := reflect.TypeOf(tblPtr).Elem() + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + name := field.Tag.Get("mmtable") + if name == "-" { + continue + } + if name == "" { + name = field.Name + } + if _, ok := scMap[name]; !ok { + return table, fmt.Errorf("Unknown column %s: %w", name, ErrUnknownColumn) + } + + targetPtrs[name] = v.Field(i).Addr().Interface() + } + + for _, sc := range schema { + target := targetPtrs[sc.Name] + if target == nil { + continue + } + bs, n, err := newBufferedSlice(buf, sc, target, capacity) + if err != nil { + return table, err + } + table = append(table, bs) + buf = buf[n:] + } + + return table, nil +} + +func (bbt bufferedTable) Lengths() []int { + lengths := make([]int, len(bbt)) + for i, ci := range bbt { + lengths[i] = ci.Len() + } + return lengths +} + +func (bbt bufferedTable) SetLengths(lengths []int) { + for i, ci := range bbt { + ci.SetLen(lengths[i]) + } +} diff --git a/bufferedtable_test.go b/bufferedtable_test.go new file mode 100644 index 0000000..21d5fc5 --- /dev/null +++ b/bufferedtable_test.go @@ -0,0 +1,233 @@ +package mmtable + +import ( + "errors" + "math/rand" + "reflect" + "testing" +) + +type TestTable struct { + Byte []byte + Bool []bool + Int8 []int8 + Int16 []int16 + Int32 []int32 + Int64 []int64 + Uint8 []uint8 + Uint16 []uint16 + Uint32 []uint32 + Uint64 []uint64 + Float32 []float32 + Float64 []float64 + Hidden []int64 `mmtable:"-"` +} + +var testSchema = Schema([]SchemaCol{ + {Name: "Byte", ElemType: ElemTypeByte}, + {Name: "Bool", ElemType: ElemTypeBool}, + {Name: "Int8", ElemType: ElemTypeInt8}, + {Name: "Int16", ElemType: ElemTypeInt16}, + {Name: "Int32", ElemType: ElemTypeInt32}, + {Name: "Int64", ElemType: ElemTypeInt64}, + {Name: "Uint8", ElemType: ElemTypeUint8}, + {Name: "Uint16", ElemType: ElemTypeUint16}, + {Name: "Uint32", ElemType: ElemTypeUint32}, + {Name: "Uint64", ElemType: ElemTypeUint64}, + {Name: "Float32", ElemType: ElemTypeFloat32}, + {Name: "Float64", ElemType: ElemTypeFloat64}, +}) + +func TestBufferedTable(t *testing.T) { + capacity := 100 + buf := make([]byte, testSchema.dataSize(capacity)) + tt := &TestTable{} + + bt, err := newBufferedTable(buf, testSchema, tt, capacity) + if err != nil { + t.Fatal(err) + } + + // Write random data to each column. + for i := 0; i < rand.Intn(capacity); i++ { + tt.Byte = append(tt.Byte, byte(rand.Int())) + } + for i := 0; i < rand.Intn(capacity); i++ { + tt.Bool = append(tt.Bool, rand.Int()%2 == 0) + } + for i := 0; i < rand.Intn(capacity); i++ { + tt.Int8 = append(tt.Int8, int8(rand.Int())) + } + for i := 0; i < rand.Intn(capacity); i++ { + tt.Int16 = append(tt.Int16, int16(rand.Int())) + } + for i := 0; i < rand.Intn(capacity); i++ { + tt.Int32 = append(tt.Int32, int32(rand.Int())) + } + for i := 0; i < rand.Intn(capacity); i++ { + tt.Int64 = append(tt.Int64, int64(rand.Int())) + } + for i := 0; i < rand.Intn(capacity); i++ { + tt.Uint8 = append(tt.Uint8, uint8(rand.Int())) + } + for i := 0; i < rand.Intn(capacity); i++ { + tt.Uint16 = append(tt.Uint16, uint16(rand.Int())) + } + for i := 0; i < rand.Intn(capacity); i++ { + tt.Uint32 = append(tt.Uint32, uint32(rand.Int())) + } + for i := 0; i < rand.Intn(capacity); i++ { + tt.Uint64 = append(tt.Uint64, uint64(rand.Int())) + } + for i := 0; i < rand.Intn(capacity); i++ { + tt.Float32 = append(tt.Float32, rand.Float32()) + } + for i := 0; i < rand.Intn(capacity); i++ { + tt.Float64 = append(tt.Float64, rand.Float64()) + } + + // Check lengths. + lengths := []int{ + len(tt.Byte), + len(tt.Bool), + len(tt.Int8), + len(tt.Int16), + len(tt.Int32), + len(tt.Int64), + len(tt.Uint8), + len(tt.Uint16), + len(tt.Uint32), + len(tt.Uint64), + len(tt.Float32), + len(tt.Float64), + } + + if !reflect.DeepEqual(bt.Lengths(), lengths) { + t.Fatal(lengths, bt.Lengths()) + } + + // Set and check lengths. + for i := range lengths { + lengths[i] = rand.Intn(capacity) + } + + bt.SetLengths(lengths) + + lengths = []int{ + len(tt.Byte), + len(tt.Bool), + len(tt.Int8), + len(tt.Int16), + len(tt.Int32), + len(tt.Int64), + len(tt.Uint8), + len(tt.Uint16), + len(tt.Uint32), + len(tt.Uint64), + len(tt.Float32), + len(tt.Float64), + } + + if !reflect.DeepEqual(bt.Lengths(), lengths) { + t.Fatal(lengths, bt.Lengths()) + } + + tt2 := &TestTable{} + bt2, err := newBufferedTable(buf, testSchema, tt2, capacity) + if err != nil { + t.Fatal(err) + } + + bt2.SetLengths(bt.Lengths()) + + if !reflect.DeepEqual(tt, tt2) { + t.Fatal(tt, tt2) + } +} + +func TestNewBufferedTable_ErrUnknownColumn(t *testing.T) { + type MyTable struct { + M [3][3]float32 + X [3]float32 `mmtable:"x"` + } + + schema := Schema{ + { + Name: "M", + ElemType: ElemTypeFloat32, + Shape: Shape(3, 3), + }, { + Name: "X", + ElemType: ElemTypeFloat32, + Shape: Shape(3), + }, + } + + capacity := 100 + buf := make([]byte, schema.dataSize(capacity)) + tt := &MyTable{} + + _, err := newBufferedTable(buf, schema, tt, capacity) + if !errors.Is(err, ErrUnknownColumn) { + t.Fatal(err) + } +} + +func TestNewBufferedTable_MissingTarget(t *testing.T) { + type MyTable struct { + M [][3][3]byte + X [][3]float32 + } + + schema := Schema{ + { + Name: "M", + ElemType: ElemTypeByte, + Shape: Shape(3, 3), + }, { + Name: "X", + ElemType: ElemTypeFloat32, + Shape: Shape(3), + }, { + Name: "Z", + ElemType: ElemTypeFloat32, + }, + } + + capacity := 100 + buf := make([]byte, schema.dataSize(capacity)) + tt := &MyTable{} + + _, err := newBufferedTable(buf, schema, tt, capacity) + if err != nil { + t.Fatal(err) + } +} + +func TestNewBufferedTable_SchemaMismatch(t *testing.T) { + type MyTable struct { + M [][3][3]byte + X []float32 + } + + schema := Schema{ + { + Name: "M", + ElemType: ElemTypeByte, + Shape: Shape(3, 3), + }, { + Name: "X", + ElemType: ElemTypeFloat32, + Shape: Shape(3), + }, + } + + capacity := 100 + buf := make([]byte, schema.dataSize(capacity)) + tt := &MyTable{} + + _, err := newBufferedTable(buf, schema, tt, capacity) + if !errors.Is(err, ErrIncorrectColType) { + t.Fatal(err) + } +} diff --git a/constants.go b/constants.go new file mode 100644 index 0000000..553ea69 --- /dev/null +++ b/constants.go @@ -0,0 +1,7 @@ +package mmtable + +const ( + Version = 1 + columnMetaSize = 64 + ColumnNameMaxLength = 64 +) diff --git a/element.go b/element.go new file mode 100644 index 0000000..46099e5 --- /dev/null +++ b/element.go @@ -0,0 +1,40 @@ +package mmtable + +import "reflect" + +type ElemType byte + +const ( + ElemTypeByte ElemType = 0 + ElemTypeBool ElemType = 1 + ElemTypeInt8 ElemType = 2 + ElemTypeInt16 ElemType = 3 + ElemTypeInt32 ElemType = 4 + ElemTypeInt64 ElemType = 5 + ElemTypeUint8 ElemType = 6 + ElemTypeUint16 ElemType = 7 + ElemTypeUint32 ElemType = 8 + ElemTypeUint64 ElemType = 9 + ElemTypeFloat32 ElemType = 10 + ElemTypeFloat64 ElemType = 11 + elemTypeMax ElemType = 12 +) + +func elemTypeIsValid(t ElemType) bool { + return t < elemTypeMax +} + +var nativeElemType = []reflect.Type{ + reflect.TypeOf(byte(0)), + reflect.TypeOf(false), + reflect.TypeOf(int8(0)), + reflect.TypeOf(int16(0)), + reflect.TypeOf(int32(0)), + reflect.TypeOf(int64(0)), + reflect.TypeOf(uint8(0)), + reflect.TypeOf(uint16(0)), + reflect.TypeOf(uint32(0)), + reflect.TypeOf(uint64(0)), + reflect.TypeOf(float32(0)), + reflect.TypeOf(float64(0)), +} diff --git a/errors.go b/errors.go new file mode 100644 index 0000000..9b55603 --- /dev/null +++ b/errors.go @@ -0,0 +1,16 @@ +package mmtable + +import "errors" + +var ( + ErrUnknownElementType = errors.New("unknown element type") + ErrZeroArraySize = errors.New("zero array size") + ErrNameTooLong = errors.New("name is too long") + ErrNameEmpty = errors.New("name is empty") + ErrNameDuplicate = errors.New("duplicate name") + + ErrCorruptSchema = errors.New("corrupt schema data") + ErrDataTruncated = errors.New("truncated data") + ErrIncorrectColType = errors.New("incorrect column type") + ErrUnknownColumn = errors.New("unknown column") +) diff --git a/schema.go b/schema.go new file mode 100644 index 0000000..bf9d27d --- /dev/null +++ b/schema.go @@ -0,0 +1,127 @@ +package mmtable + +import ( + "encoding/binary" + "fmt" +) + +func Shape(in ...uint8) [3]uint8 { + out := [3]uint8{} + if len(in) > 3 { + in = in[:3] + } + for i := range in { + if in[i] == 0 { + break + } + out[i] = in[i] + } + return out +} + +type SchemaCol struct { + ElemType ElemType + Shape [3]uint8 // Shape of each element. Zero value => scalar. + Name string // Max length is 64 bytes. +} + +func (sc SchemaCol) validate() error { + if !elemTypeIsValid(sc.ElemType) { + return fmt.Errorf("%d: %w", sc.ElemType, ErrUnknownElementType) + } + + if len(sc.Name) > ColumnNameMaxLength { + return fmt.Errorf("%s: %w", sc.Name, ErrNameTooLong) + } + + if len(sc.Name) == 0 { + return fmt.Errorf("%s: %w", sc.Name, ErrNameEmpty) + } + + return nil +} + +type Schema []SchemaCol + +func (s Schema) Validate() error { + names := map[string]bool{} + for i := range s { + if _, ok := names[s[i].Name]; ok { + return fmt.Errorf("%s: %w", s[i].Name, ErrNameDuplicate) + } + names[s[i].Name] = true + if err := s[i].validate(); err != nil { + return err + } + } + return nil +} + +func (s Schema) schemaSize() int { + return 2 + (columnMetaSize+ColumnNameMaxLength)*len(s) +} + +func (s Schema) dataSize(capacity int) (size int) { + for _, sc := range s { + t := elemTypeType(sc.ElemType, sc.Shape) + size += int(t.Size()) + } + return size * capacity +} + +// out should have the capacity of at least `s.serializedSize()`. +func (s Schema) write(out []byte) { + + // Write the number of columns. + binary.LittleEndian.PutUint16(out[:2], uint16(len(s))) + out = out[2:] + + for _, sc := range s { + out[0] = byte(sc.ElemType) + out[1] = sc.Shape[0] + out[2] = sc.Shape[1] + out[3] = sc.Shape[2] + out = out[columnMetaSize:] + + for i, b := range []byte(sc.Name) { + out[i] = b + } + + for i := len(sc.Name); i < ColumnNameMaxLength; i++ { + out[i] = 0 + } + + out = out[ColumnNameMaxLength:] + } +} + +func readSchema(buf []byte) (Schema, error) { + colCount := binary.LittleEndian.Uint16(buf[:2]) + + schema := Schema(make([]SchemaCol, colCount)) + + if len(buf) < schema.schemaSize() { + return schema, ErrCorruptSchema + } + + buf = buf[2:] + + for i := range schema { + schema[i].ElemType = ElemType(buf[0]) + schema[i].Shape[0] = buf[1] + schema[i].Shape[1] = buf[2] + schema[i].Shape[2] = buf[3] + buf = buf[columnMetaSize:] + + length := 0 + for ; length < ColumnNameMaxLength; length++ { + if buf[length] == 0 { + break + } + } + schema[i].Name = string(buf[:length]) + buf = buf[ColumnNameMaxLength:] + } + + return schema, schema.Validate() +} diff --git a/schema_test.go b/schema_test.go new file mode 100644 index 0000000..28c874e --- /dev/null +++ b/schema_test.go @@ -0,0 +1,225 @@ +package mmtable + +import ( + "errors" + "reflect" + "strings" + "testing" +) + +func TestShape(t *testing.T) { + type TestCase struct { + in []uint8 + out [3]uint8 + } + + cases := []TestCase{ + {in: []uint8{}, out: [3]uint8{0, 0, 0}}, + {in: []uint8{0}, out: [3]uint8{0, 0, 0}}, + {in: []uint8{0, 0}, out: [3]uint8{0, 0, 0}}, + {in: []uint8{0, 0, 0}, out: [3]uint8{0, 0, 0}}, + {in: []uint8{0, 0, 0, 0}, out: [3]uint8{0, 0, 0}}, + {in: []uint8{1}, out: [3]uint8{1, 0, 0}}, + {in: []uint8{0, 1}, out: [3]uint8{0, 0, 0}}, + {in: []uint8{1, 2, 3}, out: [3]uint8{1, 2, 3}}, + {in: []uint8{1, 0, 1}, out: [3]uint8{1, 0, 0}}, + } + + for _, tc := range cases { + out := Shape(tc.in...) + if !reflect.DeepEqual(out, tc.out) { + t.Fatal(out, tc.out) + } + } +} + +func TestSchemaCol_validate(t *testing.T) { + type TestCase struct { + sc SchemaCol + err error + } + + cases := []TestCase{ + { + sc: SchemaCol{ElemType: elemTypeMax, Shape: Shape(), Name: "x"}, + err: ErrUnknownElementType, + }, { + sc: SchemaCol{ + ElemType: ElemTypeBool, + Shape: Shape(), + Name: strings.Repeat("x", ColumnNameMaxLength) + "z", + }, + err: ErrNameTooLong, + }, { + sc: SchemaCol{ElemType: ElemTypeInt8, Shape: Shape(), Name: ""}, + err: ErrNameEmpty, + }, { + sc: SchemaCol{ElemType: ElemTypeInt8, Shape: Shape(), Name: "x"}, + err: nil, + }, + } + + for _, tc := range cases { + err := tc.sc.validate() + if !errors.Is(err, tc.err) { + t.Fatal(tc.sc, tc.err, err) + } + } +} + +func TestSchema_Validate(t *testing.T) { + schema := Schema([]SchemaCol{ + { + ElemType: ElemTypeInt64, + Shape: Shape(2, 2, 0), + Name: "col1", + }, + }) + + if err := schema.Validate(); err != nil { + t.Fatal(err) + } + + schema = append(schema, SchemaCol{ + ElemType: elemTypeMax, + Shape: Shape(), + Name: "col2", + }) + + if err := schema.Validate(); !errors.Is(err, ErrUnknownElementType) { + t.Fatal(err) + } + + schema[1].ElemType = ElemTypeUint32 + schema[1].Name = "col1" + + if err := schema.Validate(); !errors.Is(err, ErrNameDuplicate) { + t.Fatal(err) + } +} + +func TestSchema_schemaSize(t *testing.T) { + schema := Schema([]SchemaCol{ + { + ElemType: ElemTypeByte, + Shape: Shape(2, 0, 0), + Name: "col1", + }, { + ElemType: ElemTypeInt32, + Shape: Shape(2, 2, 2), + Name: "col2", + }, { + ElemType: ElemTypeFloat32, + Shape: Shape(), + Name: "col3", + }, + }) + + expected := 2 + 128*3 + if schema.schemaSize() != expected { + t.Fatal(schema.schemaSize(), expected) + } +} + +func TestSchema_dataSize(t *testing.T) { + schema := Schema([]SchemaCol{ + { + ElemType: ElemTypeByte, + Shape: Shape(2, 0, 0), + Name: "col1", + }, { + ElemType: ElemTypeInt32, + Shape: Shape(2, 2, 2), + Name: "col2", + }, { + ElemType: ElemTypeFloat64, + Shape: Shape(), + Name: "col3", + }, + }) + + capacity := 100 + expected := capacity * (2 + 8*4 + 8) + if schema.dataSize(capacity) != expected { + t.Fatal(schema.dataSize(capacity), expected) + } +} + +func TestSchema_write_readSchema(t *testing.T) { + schema := Schema{} + + out := make([]byte, schema.schemaSize()) + schema.write(out) + s2, err := readSchema(out) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(schema, s2) { + t.Fatal(schema, s2) + } + + schema = append(schema, SchemaCol{ + ElemType: ElemTypeByte, + Shape: Shape(2, 0, 0), + Name: "col1", + }) + + out = make([]byte, schema.schemaSize()) + schema.write(out) + s2, err = readSchema(out) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(schema, s2) { + t.Fatal(schema, s2) + } + + schema = append(schema, SchemaCol{ + ElemType: ElemTypeFloat64, + Shape: Shape(2, 2, 3), + Name: "col2", + }) + + out = make([]byte, schema.schemaSize()) + schema.write(out) + s2, err = readSchema(out) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(schema, s2) { + t.Fatal(schema, s2) + } + + schema = append(schema, SchemaCol{ + ElemType: ElemTypeUint16, + Shape: Shape(2, 0, 0), + Name: "col3", + }) + + out = make([]byte, schema.schemaSize()) + schema.write(out) + s2, err = readSchema(out) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(schema, s2) { + t.Fatal(schema, s2) + } +} + +func TestReadSchema_ErrCorruptSchema(t *testing.T) { + schema := Schema([]SchemaCol{ + { + ElemType: ElemTypeByte, + Shape: Shape(2, 0, 0), + Name: "col1", + }, + }) + + out := make([]byte, schema.schemaSize()) + schema.write(out) + out = out[:len(out)-1] + if _, err := readSchema(out); !errors.Is(err, ErrCorruptSchema) { + t.Fatal(err) + } +} diff --git a/table.go b/table.go new file mode 100644 index 0000000..2ede09b --- /dev/null +++ b/table.go @@ -0,0 +1,28 @@ +package mmtable + +import "os" + +type Table struct { + dir string // Directory where the table lives. + lockPath string // Path to lock file. + dataPath string // Path to data file. + lengthPath string // Path to length file. + lengthStagingPath string // Length file staging path for atomic writes. + lockFile *os.File // Not nil when opened for writing. + dataFile *os.File // The data file. + mapped []byte // The mmapped file data. + writable bool // True if the data file is opened for writing. + schema Schema // The schema. + cap int // The capacity of the table. + table bufferedTable // Col info in schema order. +} + +func Create(dir string, schema Schema, cap int64) (*Table, error) { + return nil, nil +} + +// TODO: Create +// TODO: Open +// TODO: OpenRW +// TODO: Close +// TODO: Sync() // Syncs data and column lengths. diff --git a/unsafe.go b/unsafe.go new file mode 100644 index 0000000..c1f0a6e --- /dev/null +++ b/unsafe.go @@ -0,0 +1,16 @@ +package mmtable + +import ( + "reflect" +) + +func elemTypeType(el ElemType, shape [3]uint8) reflect.Type { + dType := nativeElemType[el] + for _, dim := range shape { + if dim == 0 { + break + } + dType = reflect.ArrayOf(int(dim), dType) + } + return dType +} diff --git a/util_test.go b/util_test.go new file mode 100644 index 0000000..a112568 --- /dev/null +++ b/util_test.go @@ -0,0 +1,37 @@ +package mmtable + +import ( + "fmt" + "reflect" +) + +func backingDataSize(et ElemType, shape [3]uint8, capacity int) int { + t := elemTypeType(et, shape) + return int(t.Size()) * capacity +} + +func elemTypeForSlice(x interface{}) (ElemType, [3]uint8) { + // Value of the interface. + v := reflect.ValueOf(x) + + // Type of element. + t := reflect.Indirect(v).Type().Elem() + shape := [3]uint8{} + + i := 0 + for t.Kind() == reflect.Array { + shape[i] = uint8(t.Len()) + i++ + t = t.Elem() + } + + for et, tt := range nativeElemType { + if tt == t { + return ElemType(et), shape + } + } + + panic(fmt.Sprintf("Unknown type: %v", t)) + + return 255, shape +}