time/tzdata: new package

Importing the time/tzdata package will embed a copy of the IANA
timezone database into the program. This will let the program work
correctly when the timezone database is not available on the system.
It will increase the size of the binary by about 800K.

You can also build a program with -tags timetzdata to embed the
timezone database in the program being built.

This is a roll forward of CL 224588 which was rolled back due to
test failures. In this version, the test is in the time package,
not the time/tzdata package. That lets us compare the zip file
to the time/tzdata package, ensuring that we are looking at similar
versions of tzdata information.

Fixes #21881
Fixes #38013
Fixes #38017

Change-Id: I916d9d8473abe201b897cdc2bbd9168df4ad671c
Reviewed-on: https://go-review.googlesource.com/c/go/+/228101
Run-TryBot: Ian Lance Taylor <iant@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Tobias Klauser <tobias.klauser@gmail.com>
Reviewed-by: Dmitri Shuralyov <dmitshur@golang.org>

doc/go1.15.html

@@ -87,6 +87,20 @@ TODO
 
 <h2 id="library">Core library</h2>
 
+<h3 id="time/tzdata">New embedded tzdata package</h3>
+
+<p> <!-- CL 224588 -->
+  Go 1.15 includes a new package,
+  <a href="/pkg/time/tzdata/"><code>time/tzdata</code></a>,
+  that permits embedding the timezone database into a program.
+  Importing this package (as <code>import _ "time/tzdata"</code>)
+  permits the program to find timezone information even if the
+  timezone database is not available on the local system.
+  You can also embed the timezone database by building
+  with <code>-tags timetzdata</code>.
+  Either approach increases the size of the program by about 800 KB.
+</p>
+
 <p>
 TODO
 </p>

lib/time/update.bash

@@ -28,6 +28,8 @@ rm -f ../../zoneinfo.zip
 zip -0 -r ../../zoneinfo.zip *
 cd ../..
 
+go generate time/tzdata
+
 echo
 if [ "$1" = "-work" ]; then
 	echo Left workspace behind in work/.

src/go/build/deps_test.go

@@ -166,7 +166,9 @@ var pkgDeps = map[string][]string{
 		"internal/syscall/windows/registry",
 		"syscall",
 		"syscall/js",
+		"time/tzdata",
 	},
+	"time/tzdata": {"L0", "syscall"},
 
 	"internal/cfg":     {"L0"},
 	"internal/poll":    {"L0", "internal/oserror", "internal/race", "syscall", "time", "unicode/utf16", "unicode/utf8", "internal/syscall/windows", "internal/syscall/unix"},

src/time/embed.go

@@ -0,0 +1,12 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file is used with build tag timetzdata to embed tzdata into
+// the binary.
+
+// +build timetzdata
+
+package time
+
+import _ "time/tzdata"

src/time/export_test.go

@@ -38,6 +38,10 @@ var (
 	NextStdChunk           = nextStdChunk
 )
 
+func LoadFromEmbeddedTZData(zone string) (string, error) {
+	return loadFromEmbeddedTZData(zone)
+}
+
 // StdChunkNames maps from nextStdChunk results to the matched strings.
 var StdChunkNames = map[int]string{
 	0:                               "",

src/time/tzdata/generate_zipdata.go

@@ -0,0 +1,78 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+// This program generates zipdata.go from $GOROOT/lib/time/zoneinfo.zip.
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strconv"
+)
+
+// header is put at the start of the generated file.
+// The string addition avoids this file (generate_zipdata.go) from
+// matching the "generated file" regexp.
+const header = `// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+` + `// Code generated by generate_zipdata. DO NOT EDIT.
+
+// This file contains an embedded zip archive that contains time zone
+// files compiled using the code and data maintained as part of the
+// IANA Time Zone Database.
+// The IANA asserts that the data is in the public domain.
+
+// For more information, see
+// https://www.iana.org/time-zones
+// ftp://ftp.iana.org/tz/code/tz-link.htm
+// http://tools.ietf.org/html/rfc6557
+
+package tzdata
+
+const zipdata = `
+
+func main() {
+	// We should be run in the $GOROOT/src/time/tzdata directory.
+	data, err := ioutil.ReadFile("../../../lib/time/zoneinfo.zip")
+	if err != nil {
+		die("cannot find zoneinfo.zip file: %v", err)
+	}
+
+	of, err := os.Create("zipdata.go")
+	if err != nil {
+		die("%v", err)
+	}
+
+	buf := bufio.NewWriter(of)
+	buf.WriteString(header)
+
+	ds := string(data)
+	i := 0
+	const chunk = 60
+	for ; i+chunk < len(data); i += chunk {
+		if i > 0 {
+			buf.WriteRune('\t')
+		}
+		fmt.Fprintf(buf, "%s +\n", strconv.Quote(ds[i:i+chunk]))
+	}
+	fmt.Fprintf(buf, "\t%s\n", strconv.Quote(ds[i:]))
+
+	if err := buf.Flush(); err != nil {
+		die("error writing to zipdata.go: %v", err)
+	}
+	if err := of.Close(); err != nil {
+		die("error closing zipdata.go: %v", err)
+	}
+}
+
+func die(format string, args ...interface{}) {
+	fmt.Fprintf(os.Stderr, format+"\n", args...)
+	os.Exit(1)
+}

src/time/tzdata/tzdata.go

@@ -0,0 +1,109 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:generate go run generate_zipdata.go
+
+// Package tzdata provides an embedded copy of the timezone database.
+// If this package is imported anywhere in the program, then if
+// the time package cannot find tzdata files on the system,
+// it will use this embedded information.
+//
+// Importing this package will increase the size of a program by about
+// 800 KB.
+//
+// This package should normally be imported by a program's main package,
+// not by a library. Libraries normally shouldn't decide whether to
+// include the timezone database in a program.
+//
+// This package will be automatically imported if you build with
+// -tags timetzdata.
+package tzdata
+
+// The test for this package is time/tzdata_test.go.
+
+import (
+	"errors"
+	"syscall"
+	_ "unsafe" // for go:linkname
+)
+
+// registerLoadFromEmbeddedTZData is defined in package time.
+//go:linkname registerLoadFromEmbeddedTZData time.registerLoadFromEmbeddedTZData
+func registerLoadFromEmbeddedTZData(func(string) (string, error))
+
+func init() {
+	registerLoadFromEmbeddedTZData(loadFromEmbeddedTZData)
+}
+
+// get4s returns the little-endian 32-bit value at the start of s.
+func get4s(s string) int {
+	if len(s) < 4 {
+		return 0
+	}
+	return int(s[0]) | int(s[1])<<8 | int(s[2])<<16 | int(s[3])<<24
+}
+
+// get2s returns the little-endian 16-bit value at the start of s.
+func get2s(s string) int {
+	if len(s) < 2 {
+		return 0
+	}
+	return int(s[0]) | int(s[1])<<8
+}
+
+// loadFromEmbeddedTZData returns the contents of the file with the given
+// name in an uncompressed zip file, where the contents of the file can
+// be found in embeddedTzdata.
+// This is similar to time.loadTzinfoFromZip.
+func loadFromEmbeddedTZData(name string) (string, error) {
+	const (
+		zecheader = 0x06054b50
+		zcheader  = 0x02014b50
+		ztailsize = 22
+
+		zheadersize = 30
+		zheader     = 0x04034b50
+	)
+
+	z := zipdata
+
+	idx := len(z) - ztailsize
+	n := get2s(z[idx+10:])
+	idx = get4s(z[idx+16:])
+
+	for i := 0; i < n; i++ {
+		// See time.loadTzinfoFromZip for zip entry layout.
+		if get4s(z[idx:]) != zcheader {
+			break
+		}
+		meth := get2s(z[idx+10:])
+		size := get4s(z[idx+24:])
+		namelen := get2s(z[idx+28:])
+		xlen := get2s(z[idx+30:])
+		fclen := get2s(z[idx+32:])
+		off := get4s(z[idx+42:])
+		zname := z[idx+46 : idx+46+namelen]
+		idx += 46 + namelen + xlen + fclen
+		if zname != name {
+			continue
+		}
+		if meth != 0 {
+			return "", errors.New("unsupported compression for " + name + " in embedded tzdata")
+		}
+
+		// See time.loadTzinfoFromZip for zip per-file header layout.
+		idx = off
+		if get4s(z[idx:]) != zheader ||
+			get2s(z[idx+8:]) != meth ||
+			get2s(z[idx+26:]) != namelen ||
+			z[idx+30:idx+30+namelen] != name {
+			return "", errors.New("corrupt embedded tzdata")
+		}
+		xlen = get2s(z[idx+28:])
+		idx += 30 + namelen + xlen
+		return z[idx : idx+size], nil
+	}
+
+	return "", syscall.ENOENT
+}

src/time/tzdata_test.go

@@ -0,0 +1,99 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package time_test
+
+import (
+	"reflect"
+	"testing"
+	"time"
+	_ "time/tzdata"
+)
+
+var zones = []string{
+	"Asia/Jerusalem",
+	"America/Los_Angeles",
+}
+
+func TestEmbeddedTZData(t *testing.T) {
+	time.ForceZipFileForTesting(true)
+	defer time.ForceZipFileForTesting(false)
+
+	for _, zone := range zones {
+		ref, err := time.LoadLocation(zone)
+		if err != nil {
+			t.Errorf("LoadLocation(%q): %v", zone, err)
+			continue
+		}
+
+		embedded, err := time.LoadFromEmbeddedTZData(zone)
+		if err != nil {
+			t.Errorf("LoadFromEmbeddedTZData(%q): %v", zone, err)
+			continue
+		}
+		sample, err := time.LoadLocationFromTZData(zone, []byte(embedded))
+		if err != nil {
+			t.Errorf("LoadLocationFromTZData failed for %q: %v", zone, err)
+			continue
+		}
+
+		// Compare the name and zone fields of ref and sample.
+		// The tx field changes faster as tzdata is updated.
+		// The cache fields are expected to differ.
+		v1 := reflect.ValueOf(ref).Elem()
+		v2 := reflect.ValueOf(sample).Elem()
+		typ := v1.Type()
+		nf := typ.NumField()
+		found := 0
+		for i := 0; i < nf; i++ {
+			ft := typ.Field(i)
+			if ft.Name != "name" && ft.Name != "zone" {
+				continue
+			}
+			found++
+			if !equal(t, v1.Field(i), v2.Field(i)) {
+				t.Errorf("zone %s: system and embedded tzdata field %s differs", zone, ft.Name)
+			}
+		}
+		if found != 2 {
+			t.Errorf("test must be updated for change to time.Location struct")
+		}
+	}
+}
+
+// equal is a small version of reflect.DeepEqual that we use to
+// compare the values of zoneinfo unexported fields.
+func equal(t *testing.T, f1, f2 reflect.Value) bool {
+	switch f1.Type().Kind() {
+	case reflect.Slice:
+		if f1.Len() != f2.Len() {
+			return false
+		}
+		for i := 0; i < f1.Len(); i++ {
+			if !equal(t, f1.Index(i), f2.Index(i)) {
+				return false
+			}
+		}
+		return true
+	case reflect.Struct:
+		nf := f1.Type().NumField()
+		for i := 0; i < nf; i++ {
+			if !equal(t, f1.Field(i), f2.Field(i)) {
+				return false
+			}
+		}
+		return true
+	case reflect.String:
+		return f1.String() == f2.String()
+	case reflect.Bool:
+		return f1.Bool() == f2.Bool()
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		return f1.Int() == f2.Int()
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+		return f1.Uint() == f2.Uint()
+	default:
+		t.Errorf("test internal error: unsupported kind %v", f1.Type().Kind())
+		return true
+	}
+}

src/time/zoneinfo_read.go

@@ -15,6 +15,18 @@ import (
 	"syscall"
 )
 
+// registerLoadFromEmbeddedTZData is called by the time/tzdata package,
+// if it is imported.
+func registerLoadFromEmbeddedTZData(f func(string) (string, error)) {
+	loadFromEmbeddedTZData = f
+}
+
+// loadFromEmbeddedTZData is used to load a specific tzdata file
+// from tzdata information embedded in the binary itself.
+// This is set when the time/tzdata package is imported,
+// via registerLoadFromEmbeddedTzdata.
+var loadFromEmbeddedTZData func(zipname string) (string, error)
+
 // maxFileSize is the max permitted size of files read by readFile.
 // As reference, the zoneinfo.zip distributed by Go is ~350 KB,
 // so 10MB is overkill.
@@ -486,6 +498,17 @@ func loadLocation(name string, sources []string) (z *Location, firstErr error) {
 			firstErr = err
 		}
 	}
+	if loadFromEmbeddedTZData != nil {
+		zonedata, err := loadFromEmbeddedTZData(name)
+		if err == nil {
+			if z, err = LoadLocationFromTZData(name, []byte(zonedata)); err == nil {
+				return z, nil
+			}
+		}
+		if firstErr == nil && err != syscall.ENOENT {
+			firstErr = err
+		}
+	}
 	if firstErr != nil {
 		return nil, firstErr
 	}