mirror
/
go
mirror of https://github.com/golang/go.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

cmd/pack: rewrite in Go 

Replace the pack command, a C program, with a clean reimplementation in Go.
It does not need to reproduce the full feature set and it is no longer used by
the build chain, but has a role in looking inside archives created by the build
chain directly.

Since it's not in C, it is no longer build by dist, so remove it from cmd/dist and
make it a "tool" in cmd/go terminology.

Fixes #2705

R=rsc, dave, minux.ma, josharian
CC=golang-codereviews
https://golang.org/cl/52310044
This commit is contained in:
Rob Pike 2014-01-15 09:13:52 -08:00
parent b3a3afc9b7
commit fdbf3d901b
7 changed files with 674 additions and 1613 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/cmd/dist/build.c vendored
View File

@ -1300,7 +1300,6 @@ static char *buildorder[] = {
"cmd/addr2line", "cmd/addr2line",
"cmd/objdump", "cmd/objdump",
"cmd/pack",
"cmd/prof", "cmd/prof",
"cmd/cc", // must be before c "cmd/cc", // must be before c
@ -1379,7 +1378,6 @@ static char *cleantab[] = {
"cmd/gc", "cmd/gc",
"cmd/go", "cmd/go",
"cmd/objdump", "cmd/objdump",
"cmd/pack",
"cmd/prof", "cmd/prof",
"lib9", "lib9",
"libbio", "libbio",

1
src/cmd/go/pkg.go
View File

@ -309,6 +309,7 @@ var goTools = map[string]targetDir{
"cmd/fix": toTool, "cmd/fix": toTool,
"cmd/link": toTool, "cmd/link": toTool,
"cmd/nm": toTool, "cmd/nm": toTool,
"cmd/pack": toTool,
"cmd/yacc": toTool, "cmd/yacc": toTool,
"code.google.com/p/go.tools/cmd/cover": toTool, "code.google.com/p/go.tools/cmd/cover": toTool,
"code.google.com/p/go.tools/cmd/godoc": toBin, "code.google.com/p/go.tools/cmd/godoc": toBin,

5
src/cmd/pack/Makefile
View File

@ -1,5 +0,0 @@
# Copyright 2012 The Go Authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.
include ../../Make.dist

1589
src/cmd/pack/ar.c
View File

File diff suppressed because it is too large Load Diff

41
src/cmd/pack/doc.go
View File

@ -1,29 +1,36 @@
// Copyright 2009 The Go Authors. All rights reserved. // Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style // Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
// +build ignore
/* /*
Pack is a variant of the Plan 9 ar tool. The original is documented at Pack is a simple version of the traditional Unix ar tool.
It implements only the operations needed by Go.
http://plan9.bell-labs.com/magic/man2html/1/ar
It adds a special Go-specific section __.PKGDEF that collects all the
Go type information from the files in the archive; that section is
used by the compiler when importing the package during compilation.
Usage: Usage:
go tool pack [uvnbailogS][mrxtdpq][P prefix] archive files ... go tool pack op file.a [name...]
The new option 'g' causes pack to maintain the __.PKGDEF section Pack applies the operation to the archive, using the names as arguments to the operation.
as files are added to the archive.
The new option 'S' forces pack to mark the archive as safe. The operation op is given by one of these letters:
p print files from the archive
r append files (from the file system) to the archive
t list files from the archive
x extract files from the archive
For the p, t, and x commands, listing no names on the command line
causes the operation to apply to all files in the archive.
In contrast to Unix ar, the r operation always appends to the archive,
even if a file with the given name already exists in the archive. In this way
pack's r operation is more like Unix ar's rq operation.
Adding the letter v to an operation, as in pv or rv, enables verbose operation:
For the p command, each file is prefixed by the name on a line by itself.
For the r command, names are printed as files are added.
For the t command, the listing includes additional file metadata.
For the x command, names are printed as files are extracted.
The new option 'P' causes pack to remove the given prefix
from file names in the line number information in object files
that are already stored in or added to the archive.
*/ */
package main package main

397
src/cmd/pack/pack.go Normal file
View File

@ -0,0 +1,397 @@
// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main
import (
"fmt"
"io"
"log"
"os"
"strconv"
"strings"
"time"
"unicode/utf8"
)
/*
The archive format is:
First, on a line by itself
!<arch>
Then zero or more file records. Each file record has a fixed-size one-line header
followed by data bytes followed by an optional padding byte. The header is:
%-16s%-12d%-6d%-6d%-8o%-10d`
name mtime uid gid mode size
(note the trailing backquote). The %-16s here means at most 16 *bytes* of
the name, and if shorter, space padded on the right.
*/
const usageMessage = `Usage: pack op file.a [name....]
Where op is one of prtx optionally followed by v for verbose output.
For more information, run
godoc cmd/pack`
func usage() {
fmt.Fprintln(os.Stderr, usageMessage)
os.Exit(2)
}
func main() {
log.SetFlags(0)
// need "pack op archive" at least.
if len(os.Args) < 3 {
usage()
}
setOp(os.Args[1])
var ar *Archive
switch op {
case 'p':
ar = archive(os.Args[2], os.O_RDONLY, os.Args[3:])
ar.scan(ar.printContents)
case 'r':
ar = archive(os.Args[2], os.O_RDWR, os.Args[3:])
ar.scan(ar.skipContents)
ar.addFiles()
case 't':
ar = archive(os.Args[2], os.O_RDONLY, os.Args[3:])
ar.scan(ar.tableOfContents)
case 'x':
ar = archive(os.Args[2], os.O_RDONLY, os.Args[3:])
ar.scan(ar.extractContents)
default:
usage()
}
if len(ar.files) > 0 {
log.Fatalf("pack: file %q not in archive", ar.files[0])
}
}
// The unusual ancestry means the arguments are not Go-standard.
// These variables hold the decoded operation specified by the first argument.
// op holds the operation we are doing (prtx).
// verbose tells whether the 'v' option was specified.
var (
op rune
verbose bool
)
// setOp parses the operation string (first argument).
func setOp(arg string) {
for _, r := range arg {
switch r {
case 'p', 'r', 't', 'x':
if op != 0 {
// At most one can be set.
usage()
}
op = r
case 'v':
if verbose {
// Can be set only once.
usage()
}
verbose = true
default:
usage()
}
}
}
const (
arHeader = "!<arch>\n"
entryHeader = "%s%-12d%-6d%-6d%-8o%-10d`\n"
// In entryHeader the first entry, the name, is always printed as 16 bytes right-padded.
entryLen = 16 + 12 + 6 + 6 + 8 + 10 + 1 + 1
timeFormat = "Jan _2 15:04 2006"
)
// An Archive represents an open archive file. It is always scanned sequentially
// from start to end, without backing up.
type Archive struct {
fd *os.File // Open file descriptor.
files []string // Explicit list of files to be processed.
}
// archive opens (or if necessary creates) the named archive.
func archive(name string, mode int, files []string) *Archive {
fd, err := os.OpenFile(name, mode, 0)
if err != nil && mode == os.O_RDWR && os.IsNotExist(err) {
fd, err = create(name)
}
if err != nil {
log.Fatal("pack: ", err)
}
mustBeArchive(fd)
return &Archive{
fd: fd,
files: files,
}
}
// create creates and initializes an archive that does not exist.
func create(name string) (*os.File, error) {
fd, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
return nil, err
}
fmt.Fprint(fd, arHeader)
fd.Seek(0, 0)
return fd, nil
}
// mustBeArchive verifies the header of the file. It assumes the file offset
// is 0 coming in, and leaves it positioned immediately after the header.
func mustBeArchive(fd *os.File) {
buf := make([]byte, len(arHeader))
_, err := io.ReadFull(fd, buf)
if err != nil || string(buf) != arHeader {
log.Fatal("pack: file is not an archive: bad header")
}
}
// An Entry is the internal representation of the per-file header information of one entry in the archive.
type Entry struct {
name string
mtime int64
uid int
gid int
mode os.FileMode
size int64
}
func (e *Entry) String() string {
return fmt.Sprintf("%s %6d/%-6d %12d %s %s",
(e.mode & 0777).String(),
e.uid,
e.gid,
e.size,
time.Unix(e.mtime, 0).Format(timeFormat),
e.name)
}
// readMetadata reads and parses the metadata for the next entry in the archive.
func (ar *Archive) readMetadata() *Entry {
buf := make([]byte, entryLen)
_, err := io.ReadFull(ar.fd, buf)
if err == io.EOF {
// No entries left.
return nil
}
if err != nil || buf[entryLen-2] != '`' || buf[entryLen-1] != '\n' {
log.Fatal("pack: file is not an archive: bad entry")
}
entry := new(Entry)
entry.name = strings.TrimRight(string(buf[:16]), " ")
if len(entry.name) == 0 {
log.Fatal("pack: file is not an archive: bad name")
}
buf = buf[16:]
str := string(buf)
get := func(width, base, bitsize int) int64 {
v, err := strconv.ParseInt(strings.TrimRight(str[:width], " "), base, bitsize)
if err != nil {
log.Fatal("pack: file is not an archive: bad number in entry: ", err)
}
str = str[width:]
return v
}
// %-16s%-12d%-6d%-6d%-8o%-10d`
entry.mtime = get(12, 10, 64)
entry.uid = int(get(6, 10, 32))
entry.gid = int(get(6, 10, 32))
entry.mode = os.FileMode(get(8, 8, 32))
entry.size = get(10, 10, 64)
return entry
}
// scan scans the archive and executes the specified action on each entry.
// When action returns, the file offset is at the start of the next entry.
func (ar *Archive) scan(action func(*Entry)) {
for {
entry := ar.readMetadata()
if entry == nil {
break
}
action(entry)
}
}
// listEntry prints to standard output a line describing the entry.
func listEntry(ar *Archive, entry *Entry, verbose bool) {
if verbose {
fmt.Fprintf(stdout, "%s\n", entry)
} else {
fmt.Fprintf(stdout, "%s\n", entry.name)
}
}
// output copies the entry to the specified writer.
func (ar *Archive) output(entry *Entry, w io.Writer) {
n, err := io.Copy(w, io.LimitReader(ar.fd, entry.size))
if err != nil {
log.Fatal("pack: ", err)
}
if n != entry.size {
log.Fatal("pack: short file")
}
if entry.size&1 == 1 {
_, err := ar.fd.Seek(1, 1)
if err != nil {
log.Fatal("pack: ", err)
}
}
}
// skip skips the entry without reading it.
func (ar *Archive) skip(entry *Entry) {
size := entry.size
if size&1 == 1 {
size++
}
_, err := ar.fd.Seek(size, 1)
if err != nil {
log.Fatal("pack: ", err)
}
}
// match reports whether the entry matches the argument list.
// If it does, it also drops the file from the to-be-processed list.
func (ar *Archive) match(entry *Entry) bool {
if len(ar.files) == 0 {
return true
}
for i, name := range ar.files {
if entry.name == name {
copy(ar.files[i:], ar.files[i+1:])
ar.files = ar.files[:len(ar.files)-1]
return true
}
}
return false
}
// addFiles adds files to the archive. The archive is known to be
// sane and we are positioned at the end. No attempt is made
// to check for existing files.
func (ar *Archive) addFiles() {
if len(ar.files) == 0 {
usage()
}
for _, file := range ar.files {
if verbose {
fmt.Printf("%s\n", file)
}
fd, err := os.Open(file)
if err != nil {
log.Fatal("pack: ", err)
}
ar.addFile(fd)
}
ar.files = nil
}
// FileLike abstracts the few methods we need, so we can test without needing real files.
type FileLike interface {
Name() string
Stat() (os.FileInfo, error)
Read([]byte) (int, error)
Close() error
}
// addFile adds a single file to the archive
func (ar *Archive) addFile(fd FileLike) {
defer fd.Close()
// Format the entry.
// First, get its info.
info, err := fd.Stat()
if err != nil {
log.Fatal("pack: ", err)
}
// mtime, uid, gid are all zero so repeated builds produce identical output.
mtime := int64(0)
uid := 0
gid := 0
n, err := fmt.Fprintf(ar.fd, entryHeader, exactly16Bytes(info.Name()), mtime, uid, gid, info.Mode(), info.Size())
if err != nil || n != entryLen {
log.Fatal("pack: writing entry header: ", err)
}
n64, err := io.Copy(ar.fd, fd)
if err != nil {
log.Fatal("pack: writing file: ", err)
}
if n64 != info.Size() {
log.Fatal("pack: writing file: wrote %d bytes; file is size %d", n64, info.Size())
}
if info.Size()&1 == 1 {
_, err = ar.fd.Write([]byte{0})
if err != nil {
log.Fatal("pack: writing archive: ", err)
}
}
}
// exactly16Bytes truncates the string if necessary so it is at most 16 bytes long,
// then pads the result with spaces to be exactly 16 bytes.
// Fmt uses runes for its width calculation, but we need bytes in the entry header.
func exactly16Bytes(s string) string {
for len(s) > 16 {
_, wid := utf8.DecodeLastRuneInString(s)
s = s[:len(s)-wid]
}
const sixteenSpaces = " "
s += sixteenSpaces[:16-len(s)]
return s
}
// Finally, the actual commands. Each is an action.
// can be modified for testing.
var stdout io.Writer = os.Stdout
// printContents implements the 'p' command.
func (ar *Archive) printContents(entry *Entry) {
if ar.match(entry) {
if verbose {
listEntry(ar, entry, false)
}
ar.output(entry, stdout)
} else {
ar.skip(entry)
}
}
// skipContents implements the first part of the 'r' command.
// It just scans the archive to make sure it's intact.
func (ar *Archive) skipContents(entry *Entry) {
ar.skip(entry)
}
// tableOfContents implements the 't' command.
func (ar *Archive) tableOfContents(entry *Entry) {
if ar.match(entry) {
listEntry(ar, entry, verbose)
}
ar.skip(entry)
}
// extractContents implements the 'x' command.
func (ar *Archive) extractContents(entry *Entry) {
if ar.match(entry) {
if verbose {
listEntry(ar, entry, false)
}
fd, err := os.OpenFile(entry.name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, entry.mode)
if err != nil {
log.Fatal("pack: ", err)
}
ar.output(entry, fd)
fd.Close()
} else {
ar.skip(entry)
}
}

252
src/cmd/pack/pack_test.go Normal file
View File

@ -0,0 +1,252 @@
// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package main
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"testing"
"time"
"unicode/utf8"
)
func TestExactly16Bytes(t *testing.T) {
var tests = []string{
"",
"a",
"日本語",
"1234567890123456",
"12345678901234567890",
"1234567890123本語4567890",
"12345678901234日本語567890",
"123456789012345日本語67890",
"1234567890123456日本語7890",
"1234567890123456日本語7日本語890",
}
for _, str := range tests {
got := exactly16Bytes(str)
if len(got) != 16 {
t.Errorf("exactly16Bytes(%q) is %q, length %d", str, got, len(got))
}
// Make sure it is full runes.
for _, c := range got {
if c == utf8.RuneError {
t.Errorf("exactly16Bytes(%q) is %q, has partial rune", str, got)
}
}
}
}
// tmpDir creates a temporary directory and returns its name.
func tmpDir(t *testing.T) string {
name, err := ioutil.TempDir("", "pack")
if err != nil {
t.Fatal(err)
}
return name
}
// Test that we can create an archive, write to it, and get the same contents back.
// Tests the rv and then the pv command on a new archive.
func TestCreate(t *testing.T) {
dir := tmpDir(t)
defer os.RemoveAll(dir)
name := filepath.Join(dir, "pack.a")
ar := archive(name, os.O_RDWR, nil)
// Add an entry by hand.
ar.addFile(helloFile.Reset())
ar.fd.Close()
// Now check it.
ar = archive(name, os.O_RDONLY, []string{helloFile.name})
var buf bytes.Buffer
stdout = &buf
verbose = true
defer func() {
stdout = os.Stdout
verbose = false
}()
ar.scan(ar.printContents)
ar.fd.Close()
result := buf.String()
// Expect verbose output plus file contents.
expect := fmt.Sprintf("%s\n%s", helloFile.name, helloFile.contents)
if result != expect {
t.Fatalf("expected %q got %q", expect, result)
}
}
// Test that we can create an archive, put some files in it, and get back a correct listing.
// Tests the tv command.
func TestTableOfContents(t *testing.T) {
dir := tmpDir(t)
defer os.RemoveAll(dir)
name := filepath.Join(dir, "pack.a")
ar := archive(name, os.O_RDWR, nil)
// Add some entries by hand.
ar.addFile(helloFile.Reset())
ar.addFile(goodbyeFile.Reset())
ar.fd.Close()
// Now print it.
ar = archive(name, os.O_RDONLY, nil)
var buf bytes.Buffer
stdout = &buf
verbose = true
defer func() {
stdout = os.Stdout
verbose = false
}()
ar.scan(ar.tableOfContents)
ar.fd.Close()
result := buf.String()
// Expect verbose listing.
expect := fmt.Sprintf("%s\n%s\n", helloFile.Entry(), goodbyeFile.Entry())
if result != expect {
t.Fatalf("expected %q got %q", expect, result)
}
// Do it again without verbose.
verbose = false
buf.Reset()
ar = archive(name, os.O_RDONLY, nil)
ar.scan(ar.tableOfContents)
ar.fd.Close()
result = buf.String()
// Expect non-verbose listing.
expect = fmt.Sprintf("%s\n%s\n", helloFile.name, goodbyeFile.name)
if result != expect {
t.Fatalf("expected %q got %q", expect, result)
}
}
// Test that we can create an archive, put some files in it, and get back a file.
// Tests the x command.
func TestExtract(t *testing.T) {
dir := tmpDir(t)
defer os.RemoveAll(dir)
name := filepath.Join(dir, "pack.a")
ar := archive(name, os.O_RDWR, nil)
// Add some entries by hand.
ar.addFile(helloFile.Reset())
ar.addFile(goodbyeFile.Reset())
ar.fd.Close()
// Now extract one file. We chdir to the directory of the archive for simplicity.
pwd, err := os.Getwd()
if err != nil {
t.Fatal("os.Getwd: ", err)
}
err = os.Chdir(dir)
if err != nil {
t.Fatal("os.Chdir: ", err)
}
defer func() {
err := os.Chdir(pwd)
if err != nil {
t.Fatal("os.Chdir: ", err)
}
}()
ar = archive(name, os.O_RDONLY, []string{goodbyeFile.name})
ar.scan(ar.extractContents)
ar.fd.Close()
data, err := ioutil.ReadFile(goodbyeFile.name)
if err != nil {
t.Fatal(err)
}
// Expect contents of file.
result := string(data)
expect := goodbyeFile.contents
if result != expect {
t.Fatalf("expected %q got %q", expect, result)
}
}
// Fake implementation of files.
var helloFile = &FakeFile{
name: "hello",
contents: "hello world", // 11 bytes, an odd number.
mode: 0644,
}
var goodbyeFile = &FakeFile{
name: "goodbye",
contents: "Sayonara, Jim", // 13 bytes, another odd number.
mode: 0644,
}
// FakeFile implements FileLike and also os.FileInfo.
type FakeFile struct {
name string
contents string
mode os.FileMode
offset int
}
// Reset prepares a FakeFile for reuse.
func (f *FakeFile) Reset() *FakeFile {
f.offset = 0
return f
}
// FileLike methods.
func (f *FakeFile) Name() string {
// A bit of a cheat: we only have a basename, so that's also ok for FileInfo.
return f.name
}
func (f *FakeFile) Stat() (os.FileInfo, error) {
return f, nil
}
func (f *FakeFile) Read(p []byte) (int, error) {
if f.offset >= len(f.contents) {
return 0, io.EOF
}
n := copy(p, f.contents[f.offset:])
f.offset += n
return n, nil
}
func (f *FakeFile) Close() error {
return nil
}
// os.FileInfo methods.
func (f *FakeFile) Size() int64 {
return int64(len(f.contents))
}
func (f *FakeFile) Mode() os.FileMode {
return f.mode
}
func (f *FakeFile) ModTime() time.Time {
return time.Time{}
}
func (f *FakeFile) IsDir() bool {
return false
}
func (f *FakeFile) Sys() interface{} {
return nil
}
// Special helpers.
func (f *FakeFile) Entry() *Entry {
return &Entry{
name: f.name,
mtime: 0, // Defined to be zero.
uid: 0, // Ditto.
gid: 0, // Ditto.
mode: f.mode,
size: int64(len(f.contents)),
}
}