Merge branch 'm1-xpj-tree' of bajdev/mpckit into main

docs/build-plan.md

@@ -44,7 +44,7 @@ mpckit/
   testdata/
     device/                  DemoBlank, Demo02, DemoFile with their data folders (saved on the device)
     accepted/                MpckitT1, T2, T3 with their data folders (written by the prototype, loaded on the device)
-  demo-projects/             existing folder; move into testdata/device
+  (demo-projects/ moved to testdata/device in M1)
   docs/                      copies of the format doc and this plan
   Taskfile.yml               build tasks from the Wails template
 ```

docs/xpj-format-and-scope.md

@@ -1,6 +1,6 @@
 # mpckit: MPC Sample project format and 1.0 / 2.0 scope
 
-Sources: three projects saved on an MPC Sample, in `demo-projects/` of `bajdev/mpckit`, analysed 2026-11-10.
+Sources: three projects saved on an MPC Sample, in `testdata/device/` of `bajdev/mpckit` (first committed as `demo-projects/`), analysed 2026-11-10.
 
 - `DemoBlank.xpj`: new empty project.
 - `DemoFile.xpj`: 6 factory samples on pads A1-A4 and B1-B2, two sequences with recorded notes.

go.mod

@@ -0,0 +1,3 @@
+module git.monoexp.com/bajdev/mpckit
+
+go 1.24

internal/xpj/file.go

@@ -0,0 +1,135 @@
+package xpj
+
+import (
+	"bytes"
+	"compress/gzip"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+)
+
+// HeaderLines is the number of text lines before the JSON body.
+const HeaderLines = 5
+
+// Header values seen in every file saved by firmware 1.3.0.12.
+var TestedHeader = []string{"ACVS", "1.3.0.12", "SerialisableProjectData", "json", "Linux"}
+
+// File is a decoded project file.
+type File struct {
+	Header []string // the five header lines, without newlines
+	Root   *Node    // the JSON document, {"data": {...}}
+}
+
+// Decode parses a decompressed project file.
+func Decode(raw []byte) (*File, error) {
+	parts := bytes.SplitN(raw, []byte("\n"), HeaderLines+1)
+	if len(parts) != HeaderLines+1 {
+		return nil, errors.New("xpj: file is shorter than its five header lines")
+	}
+	header := make([]string, HeaderLines)
+	for i := range header {
+		header[i] = string(parts[i])
+	}
+	if header[0] != "ACVS" {
+		return nil, fmt.Errorf("xpj: not an MPC project (first line %q, want ACVS)", header[0])
+	}
+	root, err := ParseJSON(parts[HeaderLines])
+	if err != nil {
+		return nil, err
+	}
+	if root.Kind != Object {
+		return nil, fmt.Errorf("xpj: body is %s, want object", root.Kind)
+	}
+	return &File{Header: header, Root: root}, nil
+}
+
+// Encode returns the decompressed file: header lines, then the JSON body.
+func (f *File) Encode() ([]byte, error) {
+	if len(f.Header) != HeaderLines {
+		return nil, fmt.Errorf("xpj: header has %d lines, want %d", len(f.Header), HeaderLines)
+	}
+	for _, h := range f.Header {
+		if strings.ContainsAny(h, "\r\n") {
+			return nil, fmt.Errorf("xpj: header line %q contains a newline", h)
+		}
+	}
+	body, err := f.Root.MarshalJSON()
+	if err != nil {
+		return nil, err
+	}
+	var b bytes.Buffer
+	b.Grow(len(body) + 64)
+	for _, h := range f.Header {
+		b.WriteString(h)
+		b.WriteByte('\n')
+	}
+	b.Write(body)
+	return b.Bytes(), nil
+}
+
+// Read decompresses and decodes a project file.
+func Read(r io.Reader) (*File, error) {
+	zr, err := gzip.NewReader(r)
+	if err != nil {
+		return nil, fmt.Errorf("xpj: not a gzip stream: %w", err)
+	}
+	defer zr.Close()
+	raw, err := io.ReadAll(zr)
+	if err != nil {
+		return nil, fmt.Errorf("xpj: decompress: %w", err)
+	}
+	return Decode(raw)
+}
+
+// Write encodes f and writes it gzip-compressed, with the gzip header the
+// device and the reference writer use: no name, modification time 0, OS Unix.
+func (f *File) Write(w io.Writer) error {
+	raw, err := f.Encode()
+	if err != nil {
+		return err
+	}
+	zw, err := gzip.NewWriterLevel(w, 6)
+	if err != nil {
+		return err
+	}
+	zw.OS = 3
+	if _, err := zw.Write(raw); err != nil {
+		return err
+	}
+	return zw.Close()
+}
+
+// ReadFile reads a project file from disk.
+func ReadFile(path string) (*File, error) {
+	fh, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer fh.Close()
+	return Read(fh)
+}
+
+// WriteFile writes a project file to disk.
+func (f *File) WriteFile(path string) error {
+	var b bytes.Buffer
+	if err := f.Write(&b); err != nil {
+		return err
+	}
+	return os.WriteFile(path, b.Bytes(), 0o644)
+}
+
+// Data returns the "data" object that holds the whole project.
+func (f *File) Data() (*Node, error) {
+	d := f.Root.Lookup("data")
+	if d == nil || d.Kind != Object {
+		return nil, errors.New("xpj: no data object")
+	}
+	return d, nil
+}
+
+// Clone returns a deep copy of f.
+func (f *File) Clone() *File {
+	return &File{Header: append([]string(nil), f.Header...), Root: f.Root.Clone()}
+}

internal/xpj/json.go

@@ -0,0 +1,230 @@
+package xpj
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+)
+
+// ParseJSON builds a tree from one JSON document. Numbers keep their text.
+func ParseJSON(body []byte) (*Node, error) {
+	dec := json.NewDecoder(bytes.NewReader(body))
+	dec.UseNumber()
+	root, err := parseValue(dec)
+	if err != nil {
+		return nil, err
+	}
+	if _, err := dec.Token(); err != io.EOF {
+		if err == nil {
+			err = errors.New("unexpected data after the document")
+		}
+		return nil, fmt.Errorf("xpj: %w", err)
+	}
+	return root, nil
+}
+
+func parseValue(dec *json.Decoder) (*Node, error) {
+	tok, err := dec.Token()
+	if err != nil {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+		return nil, fmt.Errorf("xpj: %w", err)
+	}
+	switch t := tok.(type) {
+	case nil:
+		return NewNull(), nil
+	case bool:
+		return NewBool(t), nil
+	case json.Number:
+		return &Node{Kind: Number, Text: string(t)}, nil
+	case string:
+		return NewString(t), nil
+	case json.Delim:
+		switch t {
+		case '[':
+			n := &Node{Kind: Array, Items: []*Node{}}
+			for dec.More() {
+				v, err := parseValue(dec)
+				if err != nil {
+					return nil, err
+				}
+				n.Items = append(n.Items, v)
+			}
+			_, err := dec.Token() // ]
+			return n, wrap(err)
+		case '{':
+			n := &Node{Kind: Object, Members: []Member{}}
+			for dec.More() {
+				kt, err := dec.Token()
+				if err != nil {
+					return nil, wrap(err)
+				}
+				v, err := parseValue(dec)
+				if err != nil {
+					return nil, err
+				}
+				n.Members = append(n.Members, Member{Key: kt.(string), Value: v})
+			}
+			_, err := dec.Token() // }
+			return n, wrap(err)
+		}
+	}
+	return nil, fmt.Errorf("xpj: unexpected token %v", tok)
+}
+
+func wrap(err error) error {
+	if err == nil {
+		return nil
+	}
+	return fmt.Errorf("xpj: %w", err)
+}
+
+// MarshalJSON writes n in the device's layout: one key or value per line, no
+// indentation, ": " after keys, "," at line ends, empty containers as [] and
+// {}, no trailing newline.
+func (n *Node) MarshalJSON() ([]byte, error) {
+	var b bytes.Buffer
+	if err := writeValue(&b, n); err != nil {
+		return nil, err
+	}
+	return b.Bytes(), nil
+}
+
+func writeValue(b *bytes.Buffer, n *Node) error {
+	if n == nil {
+		return errors.New("xpj: nil node in tree")
+	}
+	switch n.Kind {
+	case Null:
+		b.WriteString("null")
+	case Bool:
+		if n.Bool {
+			b.WriteString("true")
+		} else {
+			b.WriteString("false")
+		}
+	case Number:
+		if !isNumber(n.Text) {
+			return fmt.Errorf("xpj: invalid number text %q", n.Text)
+		}
+		b.WriteString(n.Text)
+	case String:
+		writeString(b, n.Text)
+	case Array:
+		if len(n.Items) == 0 {
+			b.WriteString("[]")
+			return nil
+		}
+		b.WriteString("[\n")
+		for i, it := range n.Items {
+			if i > 0 {
+				b.WriteString(",\n")
+			}
+			if err := writeValue(b, it); err != nil {
+				return err
+			}
+		}
+		b.WriteString("\n]")
+	case Object:
+		if len(n.Members) == 0 {
+			b.WriteString("{}")
+			return nil
+		}
+		b.WriteString("{\n")
+		for i, m := range n.Members {
+			if i > 0 {
+				b.WriteString(",\n")
+			}
+			writeString(b, m.Key)
+			b.WriteString(": ")
+			if err := writeValue(b, m.Value); err != nil {
+				return err
+			}
+		}
+		b.WriteString("\n}")
+	default:
+		return fmt.Errorf("xpj: unknown node kind %d", n.Kind)
+	}
+	return nil
+}
+
+// writeString escapes like the reference writer (Python json, ensure_ascii
+// off): quote, backslash and control characters only. '<', '>', '&', '/' and
+// non-ASCII text are written as they are.
+func writeString(b *bytes.Buffer, s string) {
+	const hex = "0123456789abcdef"
+	b.WriteByte('"')
+	for i := 0; i < len(s); i++ {
+		c := s[i]
+		switch c {
+		case '"':
+			b.WriteString(`\"`)
+		case '\\':
+			b.WriteString(`\\`)
+		case '\n':
+			b.WriteString(`\n`)
+		case '\r':
+			b.WriteString(`\r`)
+		case '\t':
+			b.WriteString(`\t`)
+		case '\b':
+			b.WriteString(`\b`)
+		case '\f':
+			b.WriteString(`\f`)
+		default:
+			if c < 0x20 {
+				b.WriteString(`\u00`)
+				b.WriteByte(hex[c>>4])
+				b.WriteByte(hex[c&0xf])
+			} else {
+				b.WriteByte(c)
+			}
+		}
+	}
+	b.WriteByte('"')
+}
+
+// isNumber reports whether s is a JSON number literal.
+func isNumber(s string) bool {
+	i := 0
+	if i < len(s) && s[i] == '-' {
+		i++
+	}
+	switch {
+	case i < len(s) && s[i] == '0':
+		i++
+	case i < len(s) && s[i] >= '1' && s[i] <= '9':
+		for i < len(s) && s[i] >= '0' && s[i] <= '9' {
+			i++
+		}
+	default:
+		return false
+	}
+	if i < len(s) && s[i] == '.' {
+		i++
+		start := i
+		for i < len(s) && s[i] >= '0' && s[i] <= '9' {
+			i++
+		}
+		if i == start {
+			return false
+		}
+	}
+	if i < len(s) && (s[i] == 'e' || s[i] == 'E') {
+		i++
+		if i < len(s) && (s[i] == '+' || s[i] == '-') {
+			i++
+		}
+		start := i
+		for i < len(s) && s[i] >= '0' && s[i] <= '9' {
+			i++
+		}
+		if i == start {
+			return false
+		}
+	}
+	return i == len(s)
+}

internal/xpj/node.go

@@ -0,0 +1,244 @@
+// Package xpj reads and writes MPC Sample project files (.xpj).
+//
+// A project file is a gzip stream holding five header lines and one JSON
+// document. The JSON is held as an ordered tree: objects keep their key order
+// and numbers keep their original text, so a file that is read and written
+// back unchanged is byte-identical after decompression. The package knows
+// nothing about pads, samples or sequences.
+package xpj
+
+import (
+	"fmt"
+	"math"
+	"strconv"
+	"strings"
+)
+
+// Kind is the JSON type of a Node.
+type Kind int
+
+const (
+	Null Kind = iota
+	Bool
+	Number
+	String
+	Array
+	Object
+)
+
+func (k Kind) String() string {
+	switch k {
+	case Null:
+		return "null"
+	case Bool:
+		return "bool"
+	case Number:
+		return "number"
+	case String:
+		return "string"
+	case Array:
+		return "array"
+	case Object:
+		return "object"
+	}
+	return fmt.Sprintf("Kind(%d)", int(k))
+}
+
+// Node is one JSON value.
+type Node struct {
+	Kind Kind
+	// Text is the decoded value of a String, or the literal text of a Number.
+	Text    string
+	Bool    bool
+	Items   []*Node  // Array elements
+	Members []Member // Object members, in file order
+}
+
+// Member is one key/value pair of an object.
+type Member struct {
+	Key   string
+	Value *Node
+}
+
+// NewNull returns a null node.
+func NewNull() *Node { return &Node{Kind: Null} }
+
+// NewBool returns a boolean node.
+func NewBool(b bool) *Node { return &Node{Kind: Bool, Bool: b} }
+
+// NewString returns a string node.
+func NewString(s string) *Node { return &Node{Kind: String, Text: s} }
+
+// NewInt returns a number node written as an integer, e.g. 4.
+func NewInt(i int64) *Node { return &Node{Kind: Number, Text: strconv.FormatInt(i, 10)} }
+
+// NewFloat returns a number node for a float field. It is always written with
+// a decimal point or exponent (96.0, never 96), in the shortest form that
+// reads back as the same float64. It panics on NaN or infinity, which JSON
+// cannot hold.
+func NewFloat(f float64) *Node { return &Node{Kind: Number, Text: FormatFloat(f)} }
+
+// NewFloat32 is NewFloat for a field the device stores as a 32-bit float:
+// the value is rounded to float32 first, then printed as a float64
+// (12.4 is written as 12.399999618530273).
+func NewFloat32(f float64) *Node { return NewFloat(float64(float32(f))) }
+
+// NewArray returns an array node holding items.
+func NewArray(items ...*Node) *Node { return &Node{Kind: Array, Items: items} }
+
+// NewObject returns an empty object node.
+func NewObject() *Node { return &Node{Kind: Object} }
+
+// FormatFloat prints f the way the reference writer (Python's repr) does:
+// shortest round-trip digits, plain notation for exponents from -4 to 15
+// with at least one digit after the point, otherwise d.ddde±XX.
+func FormatFloat(f float64) string {
+	if math.IsNaN(f) || math.IsInf(f, 0) {
+		panic("xpj: cannot write NaN or infinity")
+	}
+	s := strconv.FormatFloat(f, 'e', -1, 64) // e.g. -1.2345e+01
+	mant, expStr, _ := strings.Cut(s, "e")
+	exp, _ := strconv.Atoi(expStr)
+	neg := strings.HasPrefix(mant, "-")
+	digits := strings.Replace(strings.TrimPrefix(mant, "-"), ".", "", 1)
+
+	var out string
+	if exp < -4 || exp >= 16 {
+		out = digits[:1]
+		if len(digits) > 1 {
+			out += "." + digits[1:]
+		}
+		sign := "+"
+		if exp < 0 {
+			sign, exp = "-", -exp
+		}
+		out += fmt.Sprintf("e%s%02d", sign, exp)
+	} else if exp < 0 {
+		out = "0." + strings.Repeat("0", -exp-1) + digits
+	} else if exp+1 >= len(digits) {
+		out = digits + strings.Repeat("0", exp+1-len(digits)) + ".0"
+	} else {
+		out = digits[:exp+1] + "." + digits[exp+1:]
+	}
+	if neg {
+		out = "-" + out
+	}
+	return out
+}
+
+// Int returns the value of a Number node as an integer.
+func (n *Node) Int() (int64, error) {
+	if n.Kind != Number {
+		return 0, fmt.Errorf("xpj: %s is not a number", n.Kind)
+	}
+	if i, err := strconv.ParseInt(n.Text, 10, 64); err == nil {
+		return i, nil
+	}
+	f, err := strconv.ParseFloat(n.Text, 64)
+	if err != nil || f != math.Trunc(f) || math.Abs(f) > 1<<53 {
+		return 0, fmt.Errorf("xpj: %s is not an integer", n.Text)
+	}
+	return int64(f), nil
+}
+
+// Float returns the value of a Number node.
+func (n *Node) Float() (float64, error) {
+	if n.Kind != Number {
+		return 0, fmt.Errorf("xpj: %s is not a number", n.Kind)
+	}
+	return strconv.ParseFloat(n.Text, 64)
+}
+
+// Lookup returns the value of the first member called key, or nil if n is not
+// an object or has no such member.
+func (n *Node) Lookup(key string) *Node {
+	if n == nil || n.Kind != Object {
+		return nil
+	}
+	for _, m := range n.Members {
+		if m.Key == key {
+			return m.Value
+		}
+	}
+	return nil
+}
+
+// SetKey replaces the value of the first member called key, keeping its
+// position, or appends a new member if there is none.
+func (n *Node) SetKey(key string, v *Node) error {
+	if n.Kind != Object {
+		return fmt.Errorf("xpj: cannot set key %q on %s", key, n.Kind)
+	}
+	for i := range n.Members {
+		if n.Members[i].Key == key {
+			n.Members[i].Value = v
+			return nil
+		}
+	}
+	n.Members = append(n.Members, Member{Key: key, Value: v})
+	return nil
+}
+
+// DeleteKey removes the first member called key and reports whether it was there.
+func (n *Node) DeleteKey(key string) bool {
+	if n.Kind != Object {
+		return false
+	}
+	for i, m := range n.Members {
+		if m.Key == key {
+			n.Members = append(n.Members[:i], n.Members[i+1:]...)
+			return true
+		}
+	}
+	return false
+}
+
+// Insert puts v into an array at index i (0 <= i <= len), shifting later items.
+func (n *Node) Insert(i int, v *Node) error {
+	if n.Kind != Array {
+		return fmt.Errorf("xpj: cannot insert into %s", n.Kind)
+	}
+	if i < 0 || i > len(n.Items) {
+		return fmt.Errorf("xpj: insert index %d out of range [0,%d]", i, len(n.Items))
+	}
+	n.Items = append(n.Items, nil)
+	copy(n.Items[i+1:], n.Items[i:])
+	n.Items[i] = v
+	return nil
+}
+
+// Append adds v to the end of an array.
+func (n *Node) Append(v *Node) error { return n.Insert(len(n.Items), v) }
+
+// Remove deletes the array item at index i.
+func (n *Node) Remove(i int) error {
+	if n.Kind != Array {
+		return fmt.Errorf("xpj: cannot remove from %s", n.Kind)
+	}
+	if i < 0 || i >= len(n.Items) {
+		return fmt.Errorf("xpj: remove index %d out of range [0,%d)", i, len(n.Items))
+	}
+	n.Items = append(n.Items[:i], n.Items[i+1:]...)
+	return nil
+}
+
+// Clone returns a deep copy of n.
+func (n *Node) Clone() *Node {
+	if n == nil {
+		return nil
+	}
+	c := &Node{Kind: n.Kind, Text: n.Text, Bool: n.Bool}
+	if n.Items != nil {
+		c.Items = make([]*Node, len(n.Items))
+		for i, it := range n.Items {
+			c.Items[i] = it.Clone()
+		}
+	}
+	if n.Members != nil {
+		c.Members = make([]Member, len(n.Members))
+		for i, m := range n.Members {
+			c.Members[i] = Member{Key: m.Key, Value: m.Value.Clone()}
+		}
+	}
+	return c
+}

internal/xpj/path.go

@@ -0,0 +1,139 @@
+package xpj
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+// A path names a node below another, in the form used by the format doc:
+// object keys separated by dots, array indexes in brackets, e.g.
+// "tracks[0].program.drum.instruments[4].layersv[0].sliceInfo.End".
+// Keys containing '.', '[' or ']' cannot be addressed this way; use Lookup.
+
+type step struct {
+	key   string
+	index int // -1 for a key step
+}
+
+func parsePath(path string) ([]step, error) {
+	var steps []step
+	rest := path
+	for rest != "" {
+		if rest[0] == '[' {
+			end := strings.IndexByte(rest, ']')
+			if end < 0 {
+				return nil, fmt.Errorf("xpj: path %q: missing ]", path)
+			}
+			i, err := strconv.Atoi(rest[1:end])
+			if err != nil || i < 0 {
+				return nil, fmt.Errorf("xpj: path %q: bad index %q", path, rest[1:end])
+			}
+			steps = append(steps, step{index: i})
+			rest = rest[end+1:]
+			if strings.HasPrefix(rest, ".") {
+				rest = rest[1:]
+				if rest == "" || rest[0] == '[' {
+					return nil, fmt.Errorf("xpj: path %q: empty key", path)
+				}
+			}
+			continue
+		}
+		end := strings.IndexAny(rest, ".[]")
+		if end < 0 {
+			end = len(rest)
+		}
+		if end == 0 {
+			return nil, fmt.Errorf("xpj: path %q: empty key", path)
+		}
+		steps = append(steps, step{key: rest[:end], index: -1})
+		rest = rest[end:]
+		if strings.HasPrefix(rest, "]") {
+			return nil, fmt.Errorf("xpj: path %q: unexpected ]", path)
+		}
+		if strings.HasPrefix(rest, ".") {
+			rest = rest[1:]
+			if rest == "" || rest[0] == '[' {
+				return nil, fmt.Errorf("xpj: path %q: empty key", path)
+			}
+		}
+	}
+	return steps, nil
+}
+
+func (n *Node) walk(steps []step, path string) (*Node, error) {
+	cur := n
+	for i, s := range steps {
+		if s.index < 0 {
+			next := cur.Lookup(s.key)
+			if next == nil {
+				return nil, fmt.Errorf("xpj: path %q: no key %q at %s", path, s.key, describe(steps[:i]))
+			}
+			cur = next
+			continue
+		}
+		if cur.Kind != Array {
+			return nil, fmt.Errorf("xpj: path %q: %s is %s, not an array", path, describe(steps[:i]), cur.Kind)
+		}
+		if s.index >= len(cur.Items) {
+			return nil, fmt.Errorf("xpj: path %q: index %d out of range at %s (length %d)", path, s.index, describe(steps[:i]), len(cur.Items))
+		}
+		cur = cur.Items[s.index]
+	}
+	return cur, nil
+}
+
+func describe(steps []step) string {
+	if len(steps) == 0 {
+		return "root"
+	}
+	var b strings.Builder
+	for i, s := range steps {
+		if s.index >= 0 {
+			fmt.Fprintf(&b, "[%d]", s.index)
+			continue
+		}
+		if i > 0 {
+			b.WriteByte('.')
+		}
+		b.WriteString(s.key)
+	}
+	return b.String()
+}
+
+// Get returns the node at path below n. An empty path returns n.
+func (n *Node) Get(path string) (*Node, error) {
+	steps, err := parsePath(path)
+	if err != nil {
+		return nil, err
+	}
+	return n.walk(steps, path)
+}
+
+// Set replaces the node at path below n. The parent must exist; a missing
+// final key is appended to its object, but array indexes must already exist.
+func (n *Node) Set(path string, v *Node) error {
+	steps, err := parsePath(path)
+	if err != nil {
+		return err
+	}
+	if len(steps) == 0 {
+		return fmt.Errorf("xpj: cannot Set the root")
+	}
+	parent, err := n.walk(steps[:len(steps)-1], path)
+	if err != nil {
+		return err
+	}
+	last := steps[len(steps)-1]
+	if last.index < 0 {
+		return parent.SetKey(last.key, v)
+	}
+	if parent.Kind != Array {
+		return fmt.Errorf("xpj: path %q: parent is %s, not an array", path, parent.Kind)
+	}
+	if last.index >= len(parent.Items) {
+		return fmt.Errorf("xpj: path %q: index %d out of range (length %d)", path, last.index, len(parent.Items))
+	}
+	parent.Items[last.index] = v
+	return nil
+}

internal/xpj/xpj_test.go

@@ -0,0 +1,367 @@
+package xpj
+
+import (
+	"bytes"
+	"compress/gzip"
+	"io"
+	"math"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+const testdata = "../../testdata"
+
+func decompress(t *testing.T, path string) []byte {
+	t.Helper()
+	fh, err := os.Open(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer fh.Close()
+	zr, err := gzip.NewReader(fh)
+	if err != nil {
+		t.Fatal(err)
+	}
+	raw, err := io.ReadAll(zr)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return raw
+}
+
+// firstDiff describes where two byte slices first differ.
+func firstDiff(a, b []byte) string {
+	n := min(len(a), len(b))
+	for i := 0; i < n; i++ {
+		if a[i] != b[i] {
+			lo := max(0, i-40)
+			return "at byte " + itoa(i) + ":\n got  " + quote(a[lo:min(len(a), i+40)]) + "\n want " + quote(b[lo:min(len(b), i+40)])
+		}
+	}
+	return "lengths " + itoa(len(a)) + " and " + itoa(len(b))
+}
+
+func itoa(i int) string { return NewInt(int64(i)).Text }
+func quote(b []byte) string {
+	return strings.ReplaceAll(string(b), "\n", `\n`)
+}
+
+// The M1 milestone: every device-saved project re-writes byte-identical.
+func TestRoundTripDeviceProjects(t *testing.T) {
+	for _, name := range []string{"DemoBlank", "Demo02", "DemoFile"} {
+		t.Run(name, func(t *testing.T) {
+			roundTrip(t, filepath.Join(testdata, "device", name+".xpj"))
+		})
+	}
+}
+
+// Files from the reference writer that the device accepted round-trip too.
+func TestRoundTripAcceptedProjects(t *testing.T) {
+	paths, _ := filepath.Glob(filepath.Join(testdata, "accepted", "*.xpj"))
+	more, _ := filepath.Glob(filepath.Join(testdata, "wav-formats", "*.xpj"))
+	paths = append(paths, more...)
+	if len(paths) < 5 {
+		t.Fatalf("found only %d accepted projects", len(paths))
+	}
+	for _, p := range paths {
+		t.Run(filepath.Base(p), func(t *testing.T) { roundTrip(t, p) })
+	}
+}
+
+func roundTrip(t *testing.T, path string) {
+	t.Helper()
+	want := decompress(t, path)
+	f, err := ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := f.Encode()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(got, want) {
+		t.Fatalf("re-encoded file differs %s", firstDiff(got, want))
+	}
+
+	// Through gzip and back, as an export would.
+	var z bytes.Buffer
+	if err := f.Write(&z); err != nil {
+		t.Fatal(err)
+	}
+	f2, err := Read(&z)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got2, _ := f2.Encode()
+	if !bytes.Equal(got2, want) {
+		t.Fatalf("gzip round trip differs %s", firstDiff(got2, want))
+	}
+}
+
+func TestDeviceProjectShape(t *testing.T) {
+	f, err := ReadFile(filepath.Join(testdata, "device", "DemoBlank.xpj"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if strings.Join(f.Header, "|") != strings.Join(TestedHeader, "|") {
+		t.Errorf("header = %q", f.Header)
+	}
+	d, err := f.Data()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(d.Members) != 66 {
+		t.Errorf("data has %d keys, want 66", len(d.Members))
+	}
+	if v, _ := d.Lookup("version").Int(); v != 28 {
+		t.Errorf("data.version = %d, want 28", v)
+	}
+	pads, err := d.Get("tracks[0].program.drum.instruments")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(pads.Items) != 128 {
+		t.Errorf("%d pads, want 128", len(pads.Items))
+	}
+	seq, err := d.Get("sequences[0].value")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if seq.Lookup("name").Text != "Sequence 01" {
+		t.Errorf("sequence name = %q", seq.Lookup("name").Text)
+	}
+	if bpm, _ := seq.Lookup("bpm").Float(); bpm != 128 {
+		t.Errorf("bpm = %v", bpm)
+	}
+}
+
+func TestGzipHeader(t *testing.T) {
+	f := &File{Header: TestedHeader, Root: NewObject()}
+	var b bytes.Buffer
+	if err := f.Write(&b); err != nil {
+		t.Fatal(err)
+	}
+	// ID, CM=deflate, FLG=0, MTIME=0, XFL=0, OS=3: the same ten bytes the device writes.
+	want := []byte{0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 0, 3}
+	if got := b.Bytes()[:10]; !bytes.Equal(got, want) {
+		t.Errorf("gzip header = % x, want % x", got, want)
+	}
+}
+
+func TestLayout(t *testing.T) {
+	in := `{"a": [1, 2.50, {}], "b": {"x": [], "y": "s<&>/"}, "c": null, "d": true, "e": -1e-05}`
+	n, err := ParseJSON([]byte(in))
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := n.MarshalJSON()
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := "{\n\"a\": [\n1,\n2.50,\n{}\n],\n\"b\": {\n\"x\": [],\n\"y\": \"s<&>/\"\n},\n\"c\": null,\n\"d\": true,\n\"e\": -1e-05\n}"
+	if string(got) != want {
+		t.Errorf("got\n%s\nwant\n%s", got, want)
+	}
+}
+
+func TestKeyOrderAndDuplicates(t *testing.T) {
+	n, err := ParseJSON([]byte(`{"z": 1, "a": 2, "z": 3}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, _ := n.MarshalJSON()
+	if want := "{\n\"z\": 1,\n\"a\": 2,\n\"z\": 3\n}"; string(got) != want {
+		t.Errorf("got %q", got)
+	}
+}
+
+func TestStringEscapes(t *testing.T) {
+	s := "q\" b\\ n\n t\t c\x01 é ✓"
+	n := NewString(s)
+	got, _ := n.MarshalJSON()
+	if want := `"q\" b\\ n\n t\t c\u0001 é ✓"`; string(got) != want {
+		t.Errorf("got %s want %s", got, want)
+	}
+	back, err := ParseJSON(got)
+	if err != nil || back.Text != s {
+		t.Errorf("read back %q, %v", back.Text, err)
+	}
+}
+
+func TestParseErrors(t *testing.T) {
+	for _, in := range []string{``, `{`, `{"a": }`, `[1,]`, `{} {}`, `{"a": 1} x`, `nul`} {
+		if _, err := ParseJSON([]byte(in)); err == nil {
+			t.Errorf("ParseJSON(%q) succeeded", in)
+		}
+	}
+	if _, err := Decode([]byte("ACVS\n1\n2\n3\n")); err == nil {
+		t.Error("short header accepted")
+	}
+	if _, err := Decode([]byte("XXXX\n1\n2\n3\n4\n{}")); err == nil {
+		t.Error("bad magic accepted")
+	}
+	if _, err := Decode([]byte("ACVS\n1\n2\n3\n4\n[]")); err == nil {
+		t.Error("array body accepted")
+	}
+	if _, err := Read(strings.NewReader("not gzip")); err == nil {
+		t.Error("non-gzip accepted")
+	}
+}
+
+func TestInvalidNumberText(t *testing.T) {
+	for _, s := range []string{"", "1.", ".5", "01", "+1", "1e", "true", `"1"`, "NaN", "1 "} {
+		n := &Node{Kind: Number, Text: s}
+		if _, err := n.MarshalJSON(); err == nil {
+			t.Errorf("number text %q accepted", s)
+		}
+	}
+}
+
+func TestFormatFloat(t *testing.T) {
+	// Expected values are Python's repr, which the reference writer used.
+	cases := map[float64]string{
+		96:                     "96.0",
+		20:                     "20.0",
+		0:                      "0.0",
+		math.Copysign(0, -1):   "-0.0",
+		12.399999618530273:     "12.399999618530273",
+		1e-05:                  "1e-05",
+		0.0001:                 "0.0001",
+		1e16:                   "1e+16",
+		1.5e16:                 "1.5e+16",
+		123456789012345.6:      "123456789012345.6",
+		1e15:                   "1000000000000000.0",
+		-2.5:                   "-2.5",
+		0.7079457640647888:     "0.7079457640647888",
+		5e-324:                 "5e-324",
+		1.7976931348623157e308: "1.7976931348623157e+308",
+		0.1:                    "0.1",
+		100:                    "100.0",
+		9999999999999998:       "9999999999999998.0",
+	}
+	for f, want := range cases {
+		if got := FormatFloat(f); got != want {
+			t.Errorf("FormatFloat(%v) = %q, want %q", f, got, want)
+		}
+	}
+	// Tuning +12 semitones 40 cents, as the device writes it.
+	if got := NewFloat32(12 + 40.0/100).Text; got != "12.399999618530273" {
+		t.Errorf("NewFloat32(12.4) = %s", got)
+	}
+	if got := NewInt(-1).Text; got != "-1" {
+		t.Errorf("NewInt(-1) = %s", got)
+	}
+}
+
+func TestPaths(t *testing.T) {
+	n, err := ParseJSON([]byte(`{"a": [{"b": [10, 20]}, {"c": "x"}], "d": {"e": 1.0}}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	get := func(p string) *Node {
+		t.Helper()
+		v, err := n.Get(p)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return v
+	}
+	if get("a[0].b[1]").Text != "20" || get("a[1].c").Text != "x" || get("d.e").Text != "1.0" {
+		t.Error("Get returned wrong nodes")
+	}
+	if get("") != n {
+		t.Error("empty path is not the root")
+	}
+	for _, p := range []string{"a[2]", "a[0].x", "d[0]", "d.e.f", "a[", "a[-1]", "a[x]", ".a", "a..b", "a.", "a]", "a.[0]"} {
+		if _, err := n.Get(p); err == nil {
+			t.Errorf("Get(%q) succeeded", p)
+		}
+	}
+
+	if err := n.Set("a[0].b[0]", NewInt(4)); err != nil {
+		t.Fatal(err)
+	}
+	if err := n.Set("d.e", NewFloat(96)); err != nil {
+		t.Fatal(err)
+	}
+	if err := n.Set("d.new", NewString("n")); err != nil {
+		t.Fatal(err)
+	}
+	if err := n.Set("a[5]", NewNull()); err == nil {
+		t.Error("Set past array end succeeded")
+	}
+	if err := n.Set("", NewNull()); err == nil {
+		t.Error("Set root succeeded")
+	}
+	got, _ := n.MarshalJSON()
+	want := "{\n\"a\": [\n{\n\"b\": [\n4,\n20\n]\n},\n{\n\"c\": \"x\"\n}\n],\n\"d\": {\n\"e\": 96.0,\n\"new\": \"n\"\n}\n}"
+	if string(got) != want {
+		t.Errorf("got\n%s", got)
+	}
+}
+
+func TestArrayEditsAndClone(t *testing.T) {
+	n, _ := ParseJSON([]byte(`{"s": [{"k": 0}]}`))
+	arr, _ := n.Get("s")
+	c := arr.Items[0].Clone()
+	if err := c.SetKey("k", NewInt(1)); err != nil {
+		t.Fatal(err)
+	}
+	if arr.Items[0].Lookup("k").Text != "0" {
+		t.Fatal("clone shares data with its source")
+	}
+	if err := arr.Insert(0, c); err != nil {
+		t.Fatal(err)
+	}
+	if err := arr.Append(NewInt(9)); err != nil {
+		t.Fatal(err)
+	}
+	if err := arr.Insert(5, NewNull()); err == nil {
+		t.Error("insert out of range succeeded")
+	}
+	if err := arr.Remove(1); err != nil {
+		t.Fatal(err)
+	}
+	if err := arr.Remove(2); err == nil {
+		t.Error("remove out of range succeeded")
+	}
+	got, _ := n.MarshalJSON()
+	if want := "{\n\"s\": [\n{\n\"k\": 1\n},\n9\n]\n}"; string(got) != want {
+		t.Errorf("got\n%s", got)
+	}
+
+	obj, _ := n.Get("s[0]")
+	if !obj.DeleteKey("k") || obj.DeleteKey("k") {
+		t.Error("DeleteKey")
+	}
+	if got, _ := obj.MarshalJSON(); string(got) != "{}" {
+		t.Errorf("emptied object = %s", got)
+	}
+
+	f := &File{Header: TestedHeader, Root: n}
+	fc := f.Clone()
+	fc.Header[0] = "X"
+	_ = fc.Root.SetKey("s", NewNull())
+	if f.Header[0] != "ACVS" || n.Lookup("s").Kind != Array {
+		t.Error("File.Clone shares data with its source")
+	}
+}
+
+func TestNumberAccessors(t *testing.T) {
+	for text, want := range map[string]int64{"4": 4, "-1": -1, "8212": 8212, "20.0": 20} {
+		if got, err := (&Node{Kind: Number, Text: text}).Int(); err != nil || got != want {
+			t.Errorf("Int(%s) = %d, %v", text, got, err)
+		}
+	}
+	if _, err := (&Node{Kind: Number, Text: "1.5"}).Int(); err == nil {
+		t.Error("Int(1.5) succeeded")
+	}
+	if _, err := NewString("1").Int(); err == nil {
+		t.Error("Int on a string succeeded")
+	}
+	if f, err := NewFloat32(0.1).Float(); err != nil || f != float64(float32(0.1)) {
+		t.Errorf("Float = %v, %v", f, err)
+	}
+}

pull-requests/m1-xpj-tree/m1-container-and-ordered-json-tree.md

@@ -0,0 +1,32 @@
+# M1: xpj container and ordered JSON tree
+
+Branch: `m1-xpj-tree` → `main`
+
+## What changed
+
+- New Go module `git.monoexp.com/bajdev/mpckit` (`go 1.24`).
+- New package `internal/xpj` (no Wails, no domain knowledge), per build plan 4.1:
+  - **Container** (`file.go`): gunzip, split off the five header lines, parse the body; write header + body, gzip level 6 with mtime 0, XFL 0, OS 3 (the same 10 header bytes the device writes). `ReadFile` / `WriteFile`, `Data()`, `Clone()`.
+  - **Ordered tree** (`node.go`, `json.go`): parsed token by token with `json.Decoder` + `UseNumber()`. Objects keep key order (and duplicate keys); numbers keep their original text. Written in the device layout: one key or value per line, no indentation, `": "` after keys, `[]` / `{}` for empty containers, no trailing newline, `< > & /` not escaped.
+  - **New values**: `NewInt` (`4`), `NewFloat` (always `96.0`, never `96`; Python-repr shortest form, matching the reference writer), `NewFloat32` (rounds to float32 first: 12.4 → `12.399999618530273`), plus string/bool/null/array/object.
+  - **Helpers**: `Get` / `Set` by path (`tracks[0].program.drum.instruments[4].layersv[0]`), `Lookup` / `SetKey` / `DeleteKey`, array `Insert` / `Append` / `Remove`, deep `Clone`, `Int()` / `Float()` accessors.
+- Moved `demo-projects/` to `testdata/device/` (build plan section 3 and 10.1); updated the two doc references.
+
+## Notes
+
+- Only the decompressed bytes are guaranteed identical. The compressed bytes differ from the device's (different deflate implementation); the device accepted the Python writer's gzip output, which uses the same header bytes as ours.
+- String escaping follows the reference writer (Python `json`, `ensure_ascii=False`). None of the device files contain an escape, so the device's own escaping style is unknown.
+- Writing a NaN/Inf float panics, and a Number node with invalid text fails at write time rather than producing a bad file.
+
+## Testing
+
+`go vet ./...` and `go test ./...` pass.
+
+- **M1 done-when**: `TestRoundTripDeviceProjects` reads DemoBlank, Demo02 and DemoFile and re-encodes each byte-identical to the decompressed original (about 3 MB each), both directly and through gzip and back.
+- The same passes for the device-accepted files in `testdata/accepted` (T1-T3) and `testdata/wav-formats` (W1, W2).
+- Shape check on DemoBlank: tested header, 66 data keys, `data.version` 28, 128 pads, "Sequence 01" at 128 BPM.
+- Unit tests for layout, key order, escapes, parse errors, invalid number text, float formatting against Python repr values, paths, array edits and clone independence.
+
+## Next
+
+M2: edit operations in `internal/project`, matching `MpckitT1.xpj` / `MpckitT3.xpj` and rebuilding `Demo02`.