lib/suitar: add new Go package $ TZ=UTC tar -tvf test/data/archive.dense.suitar -rw-r--r-- nobody/nobody 0 2020-02-05 06:10 artificial/0.bytes -rw-r--r-- nobody/nobody 853 2020-08-12 14:04 github-tags.json -rwxr-xr-x nobody/nobody 693 2021-05-03 06:19 hello.sh -rw-r--r-- nobody/nobody 104 2021-05-03 06:16 non-ascii/αβ.txt -rw-r--r-- nobody/nobody 151 2021-05-03 06:23 non-ascii/😻.txt -rw-r--r-- nobody/nobody 208 2020-02-05 06:10 pjw-thumbnail.png -rw-r--r-- nobody/nobody 942 2020-02-05 06:10 romeo.txt -rw-r--r-- nobody/nobody 558 2020-02-05 06:10 romeo.txt.gz
diff --git a/doc/spec/README.md b/doc/spec/README.md index 35e306f..5aa266b 100644 --- a/doc/spec/README.md +++ b/doc/spec/README.md
@@ -3,3 +3,4 @@ - [Naïve Image Formats: NIE, NII, NIA](/doc/spec/nie-spec.md) - [Random Access Compression: RAC](/doc/spec/rac-spec.md) + - [Simple Uncompressed Interchange Tape Archive: SUITAR](/doc/spec/suitar-spec.md)
diff --git a/doc/spec/suitar-spec.md b/doc/spec/suitar-spec.md new file mode 100644 index 0000000..3497930 --- /dev/null +++ b/doc/spec/suitar-spec.md
@@ -0,0 +1,220 @@ +# Simple Uncompressed Interchange Tape Archive: SUITAR + +Status: Draft (as of September 2026). There is no compatibility guarantee yet. + + +## Overview + +SUITAR is a common-denominator, trivial-to-parse interchange format for "the +result of decompressing a compressed file or unpacking an archive file", such +as ZIP, RAR, 7Z, TAR, GZIP, BZIP2, XZ, ZSTD, etc. + +SUITAR is to archive files (including `foobar.dat.xz` compressed files as an +archive containing 1 file) as [Farbfeld](https://tools.suckless.org/farbfeld/) +or [NIE](./nie-spec.md) is to image files: an uncompressed, "designed for Unix +pipes" format that is trivial to read or write in a few hundred lines of code, +ideally in a memory-safe programming language. It's a format for what the +Chromium web browser's "Rule of 2" security advice calls +[https://chromium.googlesource.com/chromium/src/+/master/docs/security/rule-of-2.md#normalization](Normalization). + + +## Subset of TAR + +SUITAR is a subset of the well-known and widely-used TAR (Tape Archive, with +GNU extensions) archive file format. Every valid SUITAR file is also a valid +TAR file. These files work with popular tools like `/usr/bin/tar` and with +popular TAR-reading libraries in a variety of programming languages. + +SUITAR is a subset of "pure TAR", not of "TAR wrapped in GZIP", "TAR wrapped in +BZIP2", etc. A separate compression step wrapping SUITAR is feasible, just like +wrapping TAR, but is out of scope of this document. + +Like TAR, SUITAR files are a sequence of independent entries (files or +directories). Independent means that the concatenation of two valid SUITAR +files is itself a valid SUITAR file. Like JSON map keys, duplicate names are +valid, although some decoders may choose to reject them. + +A file's entry does not need to be preceded by explicit entries for that file's +parent directories. + + +### Further Restrictions + +Compared to plain TAR (and refer to [the GNU TAR +manual](https://ftp.gnu.org/old-gnu/Manuals/tar-1.12/html_node/tar_123.html)), +SUITAR has further restrictions: + +- Entries are either regular files (`REGTYPE`), sparse files (`GNUTYPE_SPARSE`) + or directories (`DIRTYPE`). There is no support for hard links, symlinks, + device files or other non-standard files. +- Sparse files must be completely sparse. Their content must be one contiguous + span of NUL (zero) bytes that covers the entire file. +- File and directory names must obey the "File Name Validity" rules, below. +- These names must be encoded by a `GNUTYPE_LONGNAME` header block, regardless + of whether the name's length is over or under 100 or 255 bytes. +- File size and modTime (modification time, seconds since Unix epoch) integers + must use base-256 encoding (not base-8 octal) and must be non-negative and + less than `(1 << 53)`, which is `9007_199254_740992`. +- Mode bits are either 0o644 (`rw-r--r--`) or 0o755 (`rwxr-xr-x`), encoded in + base-8 octal (not base-256). +- UID and GID are hard-coded to 65534 (as a number, equivalent to octal + 0o0177776) and "nobody" (as a string). +- Any other fields are unused and must be NUL bytes. +- Padding bytes (as TAR uses 512-byte blocks) must be NUL bytes. + +There is no support for various TAR variants, such as "the PAX extensions to +TAR" or "the USTAR extensions to TAR", other than what's implied by the subset +of the GNU extensions that SUITAR explicitly uses. + +Encoders have no meaningful choices, bar one exception. There is only one valid +SUITAR encoding (unlike full TAR's backwards-compatible choice between base-8 +or base-256 encoding of various sufficiently small numbers) for any given file +or directory entry (its combination of type, name, size, mode, modTime and +contents). + +The one exception is that, if a file's contents are all NUL bytes (including +zero-sized files), an encoder can choose between a `REGTYPE` regular file (with +explicit NULs) or a `GNUTYPE_SPARSE` sparse file (with implicit NULs). + + +### File Name Validity + +These rules apply to both file names and directory names. + +- Names must be valid UTF-8. +- Names must not contain any ASCII control characters, including `'\n'` or + `'\x00'`, the "new line" or NUL bytes. +- Names must not contain the `'\x7F'` ASCII DEL byte. +- Names must be no longer than 4095 bytes, excluding a trailing NUL. +- Names must not be `""`, `"."` or `".."`. +- Names must not start or end with `"/"`, `"./"` or `"../"`. +- Names must not contain `"//"`, `"/./"` or `"/../"` as substrings. + +For example, when converting from ZIP (with Japanese file names) to SUITAR, it +is the SUITAR producer's responsibility, not the SUITAR consumer's, to detect +and transform Shift-JIS encoded names to equivalent and valid UTF-8. + + +## File Structure + +SUITAR files are a sequence of independent entries and each entry occupies an +integer number of 512-byte blocks: + +- 1 `GNUTYPE_LONGNAME` header block. +- 1 or more payload blocks containing the file or directory name. +- 1 `REGTYPE`, `GNUTYPE_SPARSE` or `DIRTYPE` header block. +- If `REGTYPE`, 0 or more payload blocks containing the file contents. +- If not `REGTYPE`, no further blocks. + + +### Header Blocks + +Like all blocks, each header block is 512 bytes long. Each header block also +starts with a 12-byte magic signature (that is not valid UTF-8), identifying +SUITAR version 1. There are no other versions at this time. + +The first 384 out of 512 bytes must match this template (arranged as 24 rows of +16 bytes per row, plus commentary): + + @@@@@@@@@@@@.... @@@@@@@@@@@@ = "\x13sUItAR\x00\xFE\xFDv1". + ................ + ................ + ................ + ................ + ................ + ....0000???.0177 ??? = mode. + 776.0177776.$... + ????????$...???? ???????? = physical size, ???????? = modTime. + ??????????. ?... ?????? = checksum, ? = type. + ................ + ................ + ................ + ................ + ................ + ................ + .ustar .nobody. + ................ + .........nobody. + ................ + ................ + ................ + ................ + ................ + +In this template, `@` indicates the magic signature, `.` indicates a `0x00` NUL +byte, `$` indicates a `0x80` byte and `?` indicates parts of the template that +are variable, not hard-coded. + +These `?` bytes are the 3-byte mode (`"644"` or `"755"`), physical size or +modTime as an 8-byte big-endian `uint64`, 6-byte checksum (see below) or 1-byte +type, which must be one of: + +- `'0'` for `REGTYPE`. +- `'5'` for `DIRTYPE`, in which case mode must be `"755"` and physical size + must be all zeroes. +- `'L'` for `GNUTYPE_LONGNAME`, in which case mode must be `"644"` and modTime + must be all zeroes. +- `'S'` for `GNUTYPE_SPARSE`, in which case physical size must be all zeroes + and offset and logical size (see below) must be the same number and, again, + within the half-open range `0 .. (1 << 53)`. + +The last 128 out of 512 bytes (8 rows of 16 bytes per row) must be all NUL +bytes unless the type is `GNUTYPE_SPARSE`, in which case it must match this +template (and `?` again indicates an 8-byte big-endian `uint64`): + + ..$...????????$. ???????? = sparse offset. + ................ + ................ + ................ + ................ + ................ + ...$...????????. ???????? = sparse logical size. + ................ + + +### Header Checksum + +A 512-byte header block's checksum value is simply the sum of each byte (after +converting from `uint8` to `uint32`, to avoid overflow) in the block, at +offsets in the two half-open ranges `0 .. 148` and `156 .. 512`, which excludes +the 8 bytes for the 6-byte checksum itself plus another two hard-coded bytes +`"\x00\x20"`. + +That checksum value is written as a 6-byte ASCII octal number in the header. +For example, `4853` (decimal) would be encoded as `"011365"` (octal). + + +### Payload Blocks + +Each entry has one or more payload blocks, between its two header blocks, +containing the file or directory name. The name length (including a trailing +NUL byte) is the first header block's physical size value, and must be within +the half-open range `2 .. 4096`, and so the excluding-a-trailing-NUL length +must range within `1 .. 4095`. Rounding up that including-a-trailing-NUL length +to a multiple of 512 gives the number of 512-byte payload blocks that contain +the name. All padding bytes in the name's final payload block must be NUL. + +For `REGTYPE` entries, the second header block's physical size value gives the +reconstructed file's size and rounding that up to a multiple of 512 gives the +number of 512-byte payload blocks that contain the file contents. Again, all +padding bytes in the contents' final payload block must be NUL. + +For other entries (`DIRTYPE` or `GNUTYPE_SPARSE`), there are no further payload +blocks after the second header block. + +For `GNUTYPE_SPARSE` entries, the second header block's logical size value +gives the reconstructed file's size and its contents are all NUL bytes. + + +# Reference Implementation + +The [google/wuffs](https://github.com/google/wuffs) repository, which holds +this specification document, also holds a +[suitar](https://godoc.org/github.com/google/wuffs/lib/suitar) Go package and +some `test/data/*.suitar` example files, readable by that Go package but also +by `/usr/bin/tar`. + + +--- + +Updated on September 2026.
diff --git a/lib/suitar/example_test.go b/lib/suitar/example_test.go new file mode 100644 index 0000000..39d9d83 --- /dev/null +++ b/lib/suitar/example_test.go
@@ -0,0 +1,87 @@ +// Copyright 2026 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. +// +// SPDX-License-Identifier: Apache-2.0 OR MIT + +package suitar_test + +import ( + "bytes" + "fmt" + "io" + "log" + "os" + "time" + + "github.com/google/wuffs/lib/suitar" +) + +func Example_minimal() { + // This mimics Example_minimal from the standard library's archive/tar. + // + // This package's Writer is a little more strict, rejecting an (implicit) + // zero-valued Header.Typeflag or Header.ModTime that archive/tar accepts. + // Header.Mode must also be 0o644 (or 0o755), not 0o600. + + // Create and add some files to the archive. + var buf bytes.Buffer + tw := suitar.NewWriter(&buf) + var files = []struct { + Name, Body string + }{ + {"readme.txt", "This archive contains some text files."}, + {"gopher.txt", "Gopher names:\nGeorge\nGeoffrey\nGonzo"}, + {"todo.txt", "Get animal handling license."}, + } + for _, file := range files { + hdr := &suitar.Header{ + Typeflag: suitar.TypeReg, + Name: file.Name, + Size: int64(len(file.Body)), + Mode: suitar.Mode644, + ModTime: time.Unix(0, 0), + } + if err := tw.WriteHeader(hdr); err != nil { + log.Fatal(err) + } + if _, err := tw.Write([]byte(file.Body)); err != nil { + log.Fatal(err) + } + } + if err := tw.Close(); err != nil { + log.Fatal(err) + } + + // Open and iterate through the files in the archive. + tr := suitar.NewReader(&buf) + for { + hdr, err := tr.Next() + if err == io.EOF { + break // End of archive + } + if err != nil { + log.Fatal(err) + } + fmt.Printf("Contents of %s:\n", hdr.Name) + if _, err := io.Copy(os.Stdout, tr); err != nil { + log.Fatal(err) + } + fmt.Println() + } + + // Output: + // Contents of readme.txt: + // This archive contains some text files. + // Contents of gopher.txt: + // Gopher names: + // George + // Geoffrey + // Gonzo + // Contents of todo.txt: + // Get animal handling license. +}
diff --git a/lib/suitar/suitar.go b/lib/suitar/suitar.go new file mode 100644 index 0000000..0de12fb --- /dev/null +++ b/lib/suitar/suitar.go
@@ -0,0 +1,659 @@ +// Copyright 2026 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. +// +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// ---------------- + +// Package suitar implements the SUITAR archive file format. +// +// SUITAR is a subset of the TAR archive file format (including its popular +// USTAR, PAX and GNU extensions). This package's API is a subset of the +// standard library's archive/tar package. +// +// Being a subset means that this package's implementation is about 600 lines +// of Go code, compared to archive/tar being about 3000, a factor of 5×. +// +// The SUITAR specification is at +// https://github.com/google/wuffs/blob/main/doc/spec/suitar-spec.md +package suitar + +import ( + "errors" + "io" + "strings" + "time" + "unicode/utf8" +) + +var ( + errBadFileName = errors.New("suitar: bad file name") + errBadHeader = errors.New("suitar: bad header") + errBadPadding = errors.New("suitar: bad padding") + errClosed = errors.New("suitar: closed") + errHeaderSize = errors.New("suitar: inconsistent Header.Size and Write length") + errHeaderTypeflag = errors.New("suitar: inconsistent Header.Typeflag for Write") +) + +// lenMagic makes headerBlockTemplate[:lenMagic] SUITAR's magic signature. +const lenMagic = 12 + +// headerBlockTemplate is the contents of each 512-byte header block. Each +// SUITAR archive entry (a file or directory) consists of: 1 header block (with +// typeflag typeGNULongName), the file name, 1 header block (with typeflag +// TypeReg or TypeDir or TypeGNUSparse) and the file contents. +// +// '?' bytes mean that the header block's bytes can vary at that position. +// Otherwise, the bytes are hard-coded. SUITAR is a stricter subset of TAR. +const headerBlockTemplate = "\x13sUItAR\x00\xFE\xFDv1\x00\x00\x00\x00" + + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + + "\x00\x00\x00\x000000???\x000177" + // ??? = mode bits, 0177776 = uid("nobody"). + "776\x000177776\x00\x80\x00\x00\x00" + + + "????????\x80\x00\x00\x00????" + // ???????? = physical file size, ???????? = modTime. + "??????????\x00 ?\x00\x00\x00" + // ?????? = checksum, ? = typeflag. + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + + + "\x00ustar \x00nobody\x00" + // "ustar " mid-block magic signature, "nobody" ≈ uid. + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + + "\x00\x00\x00\x00\x00\x00\x00\x00\x00nobody\x00" + // "nobody" ≈ gid. + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + + + // The 128-byte tail of this headerBlockTemplate only applies to + // TypeGNUSparse blocks. Otherwise, the final 128 bytes must be NUL. + "\x00\x00\x80\x00\x00\x00????????\x80\x00" + // ???????? = sparse anti-hole offset (and its length is 0). + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + + "\x00\x00\x00\x80\x00\x00\x00????????\x00" + // ???????? = sparse logical file size. + "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + +// Valid values for Header.Typeflag. The zero value is invalid. +const ( + TypeReg = '0' // Regular file (with contents). + TypeDir = '5' // Directory (with no contents). + TypeGNUSparse = 'S' // Sparse file (its contents are all NUL bytes). + + typeGNULongName = 'L' +) + +// Valid values for Header.Mode. The zero value is invalid. +const ( + Mode644 = int64(0o644) // "rw-r--r--" mode bits, also known as permission bits. + Mode755 = int64(0o755) // "rwxr-xr-x" mode bits, also known as permission bits. +) + +// IsValidHeaderName returns whether name is a valid Header.Name field value. +func IsValidHeaderName(name string) bool { + return (len(name) < 4096) && + (name != "") && + (name != ".") && + (name != "..") && + (name[0] != '/') && + (name[len(name)-1] != '/') && + !strings.HasPrefix(name, "./") && + !strings.HasPrefix(name, "../") && + !strings.Contains(name, "//") && + !strings.Contains(name, "/./") && + !strings.Contains(name, "/../") && + !containsASCIIControlCharactersOrDel(name) && + utf8.ValidString(name) +} + +func containsASCIIControlCharactersOrDel(s string) bool { + for i := range len(s) { + if c := s[i]; (c < 0x20) || (c == 0x7F) { + return true + } + } + return false +} + +func isAllZeroes(b []byte) bool { + for _, c := range b { + if c != 0 { + return false + } + } + return true +} + +func readFullNoEOF(r io.Reader, b []byte) (int, error) { + n, err := io.ReadFull(r, b) + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return n, err +} + +func u64le(b []byte) uint64 { + return (uint64(b[0]) << 56) | + (uint64(b[1]) << 48) | + (uint64(b[2]) << 40) | + (uint64(b[3]) << 32) | + (uint64(b[4]) << 24) | + (uint64(b[5]) << 16) | + (uint64(b[6]) << 8) | + (uint64(b[7]) << 0) +} + +func roundUp512(n uint64) uint64 { + return (n + 511) &^ 511 +} + +type block [512]byte + +func (b *block) calculateChecksum() uint32 { + checksum := uint32(0) + + for _, c := range b[0x000:0x094] { + checksum += uint32(c) + } + + // The 8 checksum bytes have value 32 when calculating the checksum itself. + checksum += 256 + + for _, c := range b[0x09C:0x200] { + checksum += uint32(c) + } + + return checksum +} + +func (b *block) setChecksum() { + checksum := b.calculateChecksum() + + b[0x09B] = ' ' + b[0x09A] = '\x00' + for i := range 6 { + b[0x099-i] = '0' + byte(checksum&7) + checksum >>= 3 + } +} + +func (b *block) isValidHeader(which int) bool { + // Compare b to the headerBlockTemplate. Blocks that aren't TypeGNUSparse + // must end in 128 NUL bytes. + end := 0x200 + if typeflag := b[0x09C]; typeflag != TypeGNUSparse { + end = 0x180 + } + for i := range end { + if c := headerBlockTemplate[i]; (c != '?') && (c != b[i]) { + return false + } + } + if (end == 0x180) && !isAllZeroes(b[0x180:0x200]) { + return false + } + + // Check the mode bits. + if (b[0x0068] == '6') && (b[0x0069] == '4') && (b[0x006A] == '4') { + // No-op. + } else if (b[0x0068] == '7') && (b[0x0069] == '5') && (b[0x006A] == '5') && (which == 1) { + // No-op. + } else { + return false + } + + // Check the checksum. + checksum := b.calculateChecksum() + for i := range 6 { + if b[0x099-i] != ('0' + byte(checksum&7)) { + return false + } + checksum >>= 3 + } + + return true +} + +// Header is a single header in a SUITAR archive. +// +// It is a subset of tar.Header from the standard library's archive/tar +// package, but it is stricter about what field values are valid. +type Header struct { + Typeflag byte + Name string + Size int64 + Mode int64 + ModTime time.Time +} + +// Valid returns whether h is valid to pass to a Writer. Specifically: +// +// - Typeflag must be one of three values (TypeReg, TypeDir, TypeGNUSparse). +// In particular, it cannot be zero. +// - Name must satisfy IsValidHeaderName. +// - Size must be non-negative. It must be 0 if Typeflag is TypeDir. +// - Mode must be one of two values (Mode644, Mode755). It must be Mode755 if +// Typeflag is TypeDir. +// - ModTime.Unix() must be a non-negative int64. In particular, ModTime must +// be on or after 1 January 1970 and so cannot be the zero time.Time (which +// is 1 January Year-1). +// - Size and ModTime.Unix() must also be less than (1 << 53). +// +// A Header returned (without error) by a Reader will always be valid. +func (h *Header) Valid() bool { + if h == nil { + return false + } + + switch h.Typeflag { + default: + return false + case TypeDir: + if (h.Size != 0) || (h.Mode != Mode755) { + return false + } + case TypeReg, TypeGNUSparse: + if (h.Mode != Mode644) && (h.Mode != Mode755) { + return false + } + } + + const maxExcl = 1 << 53 + m := h.ModTime.Unix() + return (0 <= h.Size) && (h.Size < maxExcl) && + (0 <= m) && (m < maxExcl) && + IsValidHeaderName(h.Name) +} + +// NewWriter creates a new Writer writing to w. +func NewWriter(w io.Writer) *Writer { + return &Writer{w: w} +} + +// Writer provides sequential writing of a SUITAR archive. +type Writer struct { + err error + w io.Writer + header Header + remaining int64 + bIndex int32 + block block + nameBuf [4096]byte +} + +func (w *Writer) flush() error { + if w.err != nil { + return w.err + } else if w.remaining != 0 { + w.err = errHeaderSize + return w.err + } else if w.bIndex != 0 { + clear(w.block[w.bIndex:]) + if _, err := w.w.Write(w.block[:]); err != nil { + w.err = err + return w.err + } + w.bIndex = 0 + } + + return nil +} + +// WriteHeader writes h and prepares to accept the file's contents (as w is +// also an io.Writer). +func (w *Writer) WriteHeader(h *Header) error { + if err := w.flush(); err != nil { + return err + } else if !h.Valid() { + w.err = errBadHeader + return w.err + } + w.header = *h + + n := len(w.header.Name) + copy(w.nameBuf[:], w.header.Name) + w.nameBuf[n] = 0 + + initBlock(&w.block, typeGNULongName, int64(n+1), Mode644, 0) + if _, err := w.w.Write(w.block[:]); err != nil { + w.err = err + return w.err + } + + w.remaining = int64(n + 1) + + if _, err := w.Write(w.nameBuf[:n+1]); err != nil { + w.err = err + return w.err + } else if err = w.flush(); err != nil { + w.err = err + return w.err + } + + initBlock(&w.block, w.header.Typeflag, w.header.Size, w.header.Mode, w.header.ModTime.Unix()) + if _, err := w.w.Write(w.block[:]); err != nil { + w.err = err + return w.err + } + + w.remaining = w.header.Size + if w.header.Typeflag == TypeGNUSparse { + w.remaining = 0 + } + + return nil +} + +func initBlock(b *block, typeflag byte, size int64, mode int64, modTime int64) { + // 0o0177776 octal is 65534, which is Debian's UID/GID for "nobody". + const uidForNobody = "0177776" + + clear(b[:]) + copy(b[:], headerBlockTemplate[:lenMagic]) + + modeBits := "0000644" + if mode == Mode755 { + modeBits = "0000755" + } + copy(b[0x064:], modeBits) + copy(b[0x06C:], uidForNobody) + copy(b[0x074:], uidForNobody) + + physicalSize := size + if typeflag == TypeGNUSparse { + physicalSize = 0 + } + setI64(b, 0x07C, physicalSize) + + setI64(b, 0x088, modTime) + b[0x09C] = typeflag + copy(b[0x101:], "ustar ") + copy(b[0x109:], "nobody") + copy(b[0x129:], "nobody") + + if typeflag == TypeGNUSparse { + setI64(b, 0x182, size) + setI64(b, 0x18E, 0) + setI64(b, 0x1E3, size) + } + + b.setChecksum() +} + +func setI64(b *block, offset int, value int64) { + // The 0x80 uses base-256 instead of octal, allowing file sizes >= 8GiB. + b[offset] = 0x80 + for i := range 8 { + b[(offset+11)-i] = byte(value) + value >>= 8 + } +} + +// Write satisfies io.Writer. +func (w *Writer) Write(b []byte) (int, error) { + if w.err != nil { + return 0, w.err + } else if (w.header.Typeflag != TypeReg) && (w.header.Typeflag != TypeGNUSparse) { + w.err = errHeaderTypeflag + return 0, w.err + } else if len(b) == 0 { + return 0, nil + } + + tooMuch := int64(len(b)) > w.remaining + if tooMuch { + b = b[:w.remaining] + } + + ret := 0 + for len(b) > 0 { + if w.bIndex == 0 { + split := len(b) &^ 511 + prefix, suffix := b[:split], b[split:] + + if len(prefix) > 0 { + n, err := w.w.Write(prefix) + w.remaining -= int64(n) + ret += n + if err != nil { + w.err = err + break + } + } + + if len(suffix) > 0 { + w.bIndex = int32(copy(w.block[:], suffix)) + w.remaining -= int64(w.bIndex) + ret += int(w.bIndex) + } + + b = nil + break + } + + n := copy(w.block[w.bIndex:], b) + w.bIndex += int32(n) + w.remaining -= int64(n) + ret += n + b = b[n:] + + if int(w.bIndex) < len(w.block) { + continue + } else if _, err := w.w.Write(w.block[:]); err != nil { + w.err = err + break + } + } + + if tooMuch && (w.err == nil) { + w.err = errHeaderSize + } + + return ret, w.err +} + +// Close satisfies io.Closer. It closes the entire archive, not just one entry. +func (w *Writer) Close() error { + if err := w.flush(); err != nil { + return err + } + w.err = errClosed + return nil +} + +// NewReader creates a new Reader reading from r. +func NewReader(r io.Reader) *Reader { + return &Reader{r: r} +} + +// Reader provides sequential reading of a SUITAR archive. +type Reader struct { + err error + r io.Reader + remaining int64 + numPadding int32 + sparse bool + block block + nameBuf [4096]byte +} + +// Next advances to the next entry in the SUITAR archive, preparing to read the +// file's contents (as r is also an io.Reader). +func (r *Reader) Next() (Header, error) { + if r.err != nil { + return Header{}, r.err + } else if r.remaining > 0 { + if _, err := io.Copy(io.Discard, r); err != nil { + r.err = err + return Header{}, r.err + } + } + + if _, err := io.ReadFull(r.r, r.block[:]); err != nil { + r.err = err + return Header{}, r.err + } + + nameLenInclNul, err := parseBlock0(&r.block) + if err != nil { + r.err = err + return Header{}, r.err + } + + n1 := nameLenInclNul - 1 + n2 := roundUp512(nameLenInclNul) + if _, err := readFullNoEOF(r.r, r.nameBuf[:n2]); err != nil { + r.err = err + return Header{}, r.err + } + name := string(r.nameBuf[:n1]) + if !IsValidHeaderName(name) || !isAllZeroes(r.nameBuf[n1:n2]) { + r.err = errBadFileName + return Header{}, r.err + } + + if _, err := readFullNoEOF(r.r, r.block[:0x200]); err != nil { + return Header{}, err + } + typeflag, size, mode, modTime, err := parseBlock1(&r.block) + if err != nil { + return Header{}, err + } + + r.remaining = size + r.numPadding = int32(roundUp512(uint64(size)) - uint64(size)) + r.sparse = typeflag == TypeGNUSparse + + return Header{ + Typeflag: typeflag, + Name: name, + Size: size, + Mode: mode, + ModTime: time.Unix(modTime, 0), + }, nil + +} + +func parseBlock0(b *block) (uint64, error) { + if !b.isValidHeader(0) { + return 0, errBadHeader + } + + size := u64le(b[0x080:]) + modTime := u64le(b[0x08C:]) + if (size < 2) || (4097 <= size) { + return 0, errBadFileName + } else if (modTime != 0) || (b[0x09C] != 'L') { + return 0, errBadHeader + } + + return size, nil +} + +func parseBlock1(b *block) (byte, int64, int64, int64, error) { + if !b.isValidHeader(1) { + return 0, 0, 0, 0, errBadHeader + } + + typeflag := b[0x09C] + if (typeflag != TypeReg) && (typeflag != TypeDir) && (typeflag != TypeGNUSparse) { + return 0, 0, 0, 0, errBadHeader + } + + mode := Mode644 + if b[0x068] == '7' { + mode = Mode755 + } + size := int64(u64le(b[0x080:])) + if size < 0 { + return 0, 0, 0, 0, errBadHeader + } + modTime := int64(u64le(b[0x08C:])) + if modTime < 0 { + return 0, 0, 0, 0, errBadHeader + } + + if typeflag == TypeDir { + if (size != 0) || (mode != Mode755) { + return 0, 0, 0, 0, errBadHeader + } + } else if typeflag == TypeGNUSparse { + if size != 0 { + return 0, 0, 0, 0, errBadHeader + } + size0 := int64(u64le(b[0x186:])) + size1 := int64(u64le(b[0x1E7:])) + if (size0 != size1) || (size1 < 0) { + return 0, 0, 0, 0, errBadHeader + } + size = size1 + } + + return typeflag, size, mode, modTime, nil +} + +// Read satisfies io.Reader. +func (r *Reader) Read(b []byte) (int, error) { + if r.err != nil { + return 0, r.err + } else if r.remaining == 0 { + return 0, io.EOF + } else if len(b) == 0 { + return 0, nil + } + + b = b[:int(min(r.remaining, int64(len(b))))] + if r.sparse { + n := len(b) + clear(b) + r.remaining -= int64(n) + if r.remaining == 0 { + return n, io.EOF + } + return n, nil + } + + n, err := r.r.Read(b) + r.remaining -= int64(n) + if r.remaining == 0 { + if r.numPadding > 0 { + padding := r.block[:r.numPadding] + _, readFullErr := readFullNoEOF(r.r, padding) + if err == nil { + err = readFullErr + if err == nil { + for _, c := range padding { + if c != 0 { + err = errBadPadding + break + } + } + } + } + r.numPadding = 0 + } + if (err == nil) || (err == io.EOF) { + return n, io.EOF + } + + } else if err == io.EOF { + err = io.ErrUnexpectedEOF + } + + r.err = err + return n, r.err +}
diff --git a/lib/suitar/suitar_test.go b/lib/suitar/suitar_test.go new file mode 100644 index 0000000..7fed0f5 --- /dev/null +++ b/lib/suitar/suitar_test.go
@@ -0,0 +1,186 @@ +// Copyright 2026 The Wuffs Authors. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. +// +// SPDX-License-Identifier: Apache-2.0 OR MIT + +package suitar + +import ( + "archive/tar" + "bytes" + "fmt" + "hash/crc32" + "io" + "os" + "testing" +) + +type crcWriter uint32 + +func (c *crcWriter) Write(b []byte) (int, error) { + state := uint32(*c) + state = crc32.Update(state, crc32.IEEETable, b) + *c = crcWriter(state) + return len(b), nil +} + +func testWriter(tt *testing.T, sparse bool) { + f, err := os.Open("../../test/data/archive.tar") + if err != nil { + tt.Fatalf("os.Open: %v", err) + } + defer f.Close() + + // Convert from t (tar, using the standard library) to s (suitar, using + // this package). + buf := bytes.Buffer{} + sWriter := NewWriter(&buf) + for tReader := tar.NewReader(f); ; { + tHeader, err := tReader.Next() + if err == io.EOF { + break + } else if err != nil { + tt.Fatalf("Next: %v", err) + } + + sHeader := &Header{ + Typeflag: tHeader.Typeflag, + Name: tHeader.Name, + Size: tHeader.Size, + Mode: tHeader.Mode, + ModTime: tHeader.ModTime, + } + if sparse && (sHeader.Typeflag == TypeReg) { + sHeader.Typeflag = TypeGNUSparse + } + if err := sWriter.WriteHeader(sHeader); err != nil { + tt.Fatalf("WriteHeader: %v", err) + } + + dstWriter := (io.Writer)(sWriter) + if sparse { + dstWriter = io.Discard + } + if _, err := io.Copy(dstWriter, tReader); err != nil { + tt.Fatalf("io.Copy: %v", err) + } + } + + if err := sWriter.Close(); err != nil { + tt.Fatalf("Close: %v", err) + } + + got := buf.Bytes() + wantFilename := "../../test/data/archive" + if sparse { + wantFilename += ".sparse.suitar" + } else { + wantFilename += ".dense.suitar" + } + want, err := os.ReadFile(wantFilename) + if err != nil { + tt.Fatalf("os.ReadFile: %v", err) + } + + if !bytes.Equal(got, want) { + tt.Fatalf("did not recreate golden test file") + } +} + +func testReader(tt *testing.T, sparse bool, ignore bool) { + filename, wantTypeflag := "../../test/data/archive.dense.suitar", " T:'0'" + if sparse { + filename, wantTypeflag = "../../test/data/archive.sparse.suitar", " T:'S'" + } + + wantChecksums := []string(nil) + if ignore { + wantChecksums = []string{ + "C:0x00000000", + "C:0x00000000", + "C:0x00000000", + "C:0x00000000", + "C:0x00000000", + "C:0x00000000", + "C:0x00000000", + "C:0x00000000", + } + + } else if sparse { + wantChecksums = []string{ + "C:0x00000000", + "C:0xB69F8E37", + "C:0x73FF3CAE", + "C:0xD71F022F", + "C:0xC446EAB8", + "C:0x5F228EB9", + "C:0x7BA7D011", + "C:0x48792A7F", + } + + } else { + wantChecksums = []string{ + "C:0x00000000", + "C:0xFEDD8F35", + "C:0x87EE5E05", + "C:0x703E9270", + "C:0xC37CB538", + "C:0x2B0B23B0", + "C:0xABE507EF", + "C:0x67FABE9C", + } + } + + want := "" + + wantChecksums[0] + wantTypeflag + " S:0x0000 M:644 MT:0x5E3A5C50 N:artificial/0.bytes\n" + + wantChecksums[1] + wantTypeflag + " S:0x0355 M:644 MT:0x5F33F6E6 N:github-tags.json\n" + + wantChecksums[2] + wantTypeflag + " S:0x02B5 M:755 MT:0x608F960B N:hello.sh\n" + + wantChecksums[3] + wantTypeflag + " S:0x0068 M:644 MT:0x608F954D N:non-ascii/αβ.txt\n" + + wantChecksums[4] + wantTypeflag + " S:0x0097 M:644 MT:0x608F96C7 N:non-ascii/😻.txt\n" + + wantChecksums[5] + wantTypeflag + " S:0x00D0 M:644 MT:0x5E3A5C50 N:pjw-thumbnail.png\n" + + wantChecksums[6] + wantTypeflag + " S:0x03AE M:644 MT:0x5E3A5C50 N:romeo.txt\n" + + wantChecksums[7] + wantTypeflag + " S:0x022E M:644 MT:0x5E3A5C50 N:romeo.txt.gz\n" + + "" + + f, err := os.Open(filename) + if err != nil { + tt.Fatalf("os.Open: %v", err) + } + defer f.Close() + + buf := bytes.Buffer{} + for r := NewReader(f); ; { + h, err := r.Next() + if err == io.EOF { + break + } else if err != nil { + tt.Fatalf("Next: %v", err) + } + + checksum := crcWriter(0) + if !ignore { + if _, err := io.Copy(&checksum, r); err != nil { + tt.Fatalf("io.Copy: %v", err) + } + } + + fmt.Fprintf(&buf, "C:0x%08X T:'%c' S:0x%04X M:%3o MT:0x%08X N:%s\n", + checksum, h.Typeflag, h.Size, h.Mode, h.ModTime.Unix(), h.Name) + } + + if got := buf.String(); got != want { + tt.Fatalf("\ngot:\n%s\nwant:\n%s", got, want) + } +} + +func TestWriterDense(tt *testing.T) { testWriter(tt, false) } +func TestWriterSparse(tt *testing.T) { testWriter(tt, true) } +func TestReaderDenseCheck(tt *testing.T) { testReader(tt, false, false) } +func TestReaderDenseIgnore(tt *testing.T) { testReader(tt, false, true) } +func TestReaderSparseCheck(tt *testing.T) { testReader(tt, true, false) } +func TestReaderSparseIgnore(tt *testing.T) { testReader(tt, true, true) }
diff --git a/test/data/archive.dense.suitar b/test/data/archive.dense.suitar new file mode 100644 index 0000000..c69ee7c --- /dev/null +++ b/test/data/archive.dense.suitar Binary files differ
diff --git a/test/data/archive.sparse.suitar b/test/data/archive.sparse.suitar new file mode 100644 index 0000000..72ec23d --- /dev/null +++ b/test/data/archive.sparse.suitar Binary files differ