vendor: update all dependencies to latest versions

This commit is contained in:
Nick Craig-Wood
2018-01-16 13:20:59 +00:00
parent 8e83fb6fb9
commit 7d3a17725d
4878 changed files with 1974213 additions and 201199 deletions
@@ -0,0 +1 @@
generator
+345
View File
@@ -0,0 +1,345 @@
// Copyright 2017 Microsoft Corporation and contributors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
// generator scours the github.com/Azure/azure-rest-api-specs repository in search for
// REST functionality that has Go configuration, then generates packages accordingly.
// This tool was developed with the intention that it be an internal tool for the
// Azure-SDK-for-Go team, but the usage of this package for public scenarios is in no
// way prohibited. For example, have a fork of https://github.com/Azure/autorest.go?
// Generate out and SDK with the same shape as our SDK, but using your patterns using
// this tool.
//
// Given that this code was developed as an internal tool, troubles with it that do not
// pertain to our usage of it may be slow to be fixed. Pull Requests are welcome though,
// feel free to contribute.
package main
import (
"bytes"
"flag"
"fmt"
"io/ioutil"
"log"
"os"
"os/exec"
"path"
"path/filepath"
"regexp"
"strings"
"time"
"github.com/marstr/collection"
)
const (
defaultGenVer = "@microsoft.azure/autorest.go@v2.1.62"
)
var (
targetFiles collection.Enumerable
packageVersion string
outputBase string
logDirBase string
autorestVer string
errLog *log.Logger
statusLog *log.Logger
dryRun bool
)
// version should be set by the linker when compiling this program by providing the following arguments:
// -X main.version=<version>
//
// If installing the generator in your machine, that means running the following the command:
// go install -ldflags "-X main.version=<version>"
//
// The reason this is not controlled in source, is to allow for the git commit SHA1 to be used as the
// version of the string. To retrieve the currently checked-out git commit SHA1, you can use the command:
// git rev-parse HEAD
//
// If this value is set, it is recommended that it be the value of the SHA1 git commit identifier for the
// source that was used at build time.
var version string
func main() {
start := time.Now()
type generationTuple struct {
fileName string
packageName string
outputFolder string
}
literateFiles := collection.Where(targetFiles, func(subject interface{}) bool {
return strings.EqualFold(path.Base(subject.(string)), "README.md")
})
tuples := collection.SelectMany(literateFiles,
// The following function compiles the regexp which finds Go related package tags in a literate file, and creates a collection.Unfolder.
// This function has been declared this way so that the relatively expensive act of compiling a regexp is only done once.
func() collection.Unfolder {
const patternText = "```" + `\s+yaml\s+\$\(\s*tag\s*\)\s*==\s*'([\d\w\-\.]+)'\s*&&\s*\$\(go\)[\w\d\-\s:]*\s+output-folder:\s+\$\(go-sdk-folder\)([\w\d\-_\\/\.]+)\s+[\w\d\-\s:]*` + "```"
goConfigPattern := regexp.MustCompile(patternText)
// This function is a collection.Unfolder which takes a literate file as a path, and retrieves all configuration which applies to a package tag and Go.
return func(subject interface{}) collection.Enumerator {
results := make(chan interface{})
go func() {
defer close(results)
targetContents, err := ioutil.ReadFile(subject.(string))
if err != nil {
errLog.Printf("Skipping %q because: %v", subject.(string), err)
return
}
matches := goConfigPattern.FindAllStringSubmatch(string(targetContents), -1)
if len(matches) == 0 {
statusLog.Printf("Skipping %q because there were no package tags with go configuration found.", subject.(string))
} else {
for _, submatch := range matches {
results <- generationTuple{
fileName: subject.(string),
packageName: normalizePath(submatch[1]),
outputFolder: normalizePath(submatch[2]),
}
}
}
}()
return results
}
}())
if dryRun {
for entry := range tuples.Enumerate(nil) {
tuple := entry.(generationTuple)
fmt.Printf("%q in %q to %q\n", tuple.packageName, tuple.fileName, tuple.outputFolder)
}
} else {
var generatedCount, formattedCount, builtCount, vettedCount uint
done := make(chan struct{})
// Call AutoRest for each of the tags in each of the literate files, generating a Go package for calling that service.
generated := tuples.Enumerate(done).ParallelSelect(func(subject interface{}) interface{} {
tuple := subject.(generationTuple)
args := []string{
tuple.fileName,
"--go",
fmt.Sprintf("--go-sdk-folder='%s'", outputBase),
"--verbose",
"--tag=" + tuple.packageName,
"--use=" + autorestVer,
}
if packageVersion != "" {
args = append(args, fmt.Sprintf("--package-version='%s'", packageVersion))
args = append(args, fmt.Sprintf("--user-agent='Azure-SDK-For-Go/%s services'", packageVersion))
}
logFileLoc := filepath.Join(logDirBase, tuple.outputFolder)
err := os.MkdirAll(logFileLoc, os.ModePerm)
if err != nil {
errLog.Printf("Could not create log directory %q", logFileLoc)
return nil
}
logFile, err := os.Create(filepath.Join(logFileLoc, "autorestLog.txt"))
if err != nil {
errLog.Printf("Could not create log file %q for AutoRest generating from: %q in %q", logFileLoc, tuple.packageName, tuple.fileName)
return nil
}
commandText := new(bytes.Buffer)
fmt.Fprint(commandText, "Executing Command: \"")
fmt.Fprint(commandText, "autorest ")
for _, a := range args {
fmt.Fprint(commandText, a)
fmt.Fprint(commandText, " ")
}
commandText.Truncate(commandText.Len() - 1)
fmt.Fprint(commandText, `"`)
fmt.Fprintln(logFile, commandText.String())
genProc := exec.Command("autorest", args...)
genProc.Stdout = logFile
genProc.Stderr = logFile
err = genProc.Run()
if err != nil {
fmt.Fprintln(logFile, "Autorest Exectution Error: ", err)
return nil
}
generatedCount++
return path.Join(outputBase, tuple.outputFolder)
}).Where(isntNil)
// Take all of the generated packages and reformat them to adhere to Go's guidelines.
formatted := generated.Select(func(subject interface{}) (result interface{}) {
err := exec.Command("gofmt", "-w", subject.(string)).Run()
if err == nil {
formattedCount++
result = subject
} else {
errLog.Printf("Failed to format: %q", subject.(string))
}
return
}).Where(isntNil)
// Build all of the packages as a sanity check.
built := formatted.Select(func(subject interface{}) (result interface{}) {
pkgName := strings.TrimPrefix(trimGoPath(subject.(string)), "/src/")
err := exec.Command("go", "build", pkgName).Run()
if err == nil {
builtCount++
return pkgName
}
errLog.Printf("Failed to build: %q", pkgName)
return nil
}).Where(isntNil)
vetted := built.Select(func(subject interface{}) interface{} {
err := exec.Command("go", "vet", subject.(string)).Run()
if err == nil {
vettedCount++
return subject
}
errLog.Printf("Failed to vet: %q", subject.(string))
return nil
}).Where(isntNil)
// Turn the crank. This loop forces evaluation of the chain of enumerators that were built up
// in the code above.
for range vetted {
// Intenionally Left Blank
}
fmt.Println("Execution Time: ", time.Now().Sub(start))
fmt.Println("Generated: ", generatedCount)
fmt.Println("Formatted: ", formattedCount)
fmt.Println("Built: ", builtCount)
fmt.Println("Vetted: ", vettedCount)
close(done)
}
}
func init() {
var useRecursive, useStatus bool
const automaticLogDirValue = "temp"
const logDirUsage = "The root directory where all logs can be found. If the value `" + automaticLogDirValue + "` is supplied, a randomly named directory will be generated in the $TEMP directory."
flag.BoolVar(&useRecursive, "r", false, "Recursively traverses the directories specified looking for literate files.")
flag.StringVar(&outputBase, "o", getDefaultOutputBase(), "The root directory to use for the output of generated files. i.e. The value to be treated as the go-sdk-folder when AutoRest is called.")
flag.BoolVar(&useStatus, "v", false, "Print status messages as generation takes place.")
flag.BoolVar(&dryRun, "p", false, "Preview which packages would be generated instead of actaully calling autorest.")
flag.StringVar(&packageVersion, "version", "", "The version that should be stamped on this SDK. This should be a semver.")
flag.StringVar(&logDirBase, "l", getDefaultOutputBase(), logDirUsage)
flag.StringVar(&autorestVer, "a", defaultGenVer, "The version of the AutoRest Go code generator to use, defaults to `"+defaultGenVer+"`.")
// Override the default usage message, printing to stderr as the default one would.
flag.Usage = func() {
fmt.Fprintln(os.Stderr, "generator [options] [target Literate files]...")
flag.PrintDefaults()
}
flag.Parse()
statusWriter := ioutil.Discard
if useStatus {
statusWriter = os.Stdout
}
statusLog = log.New(statusWriter, "[STATUS] ", 0)
errLog = log.New(os.Stderr, "[ERROR] ", 0)
if logDirBase == automaticLogDirValue {
var err error
logDirBase, err = ioutil.TempDir("", "az-go-sdk-logs")
logDirBase = normalizePath(logDirBase)
if err == nil {
statusLog.Print("Generation logs can be found at: ", logDirBase)
} else {
errLog.Print("Logging disabled. Could not create directory: ", logDirBase)
}
}
targetFiles = collection.AsEnumerable(flag.Args())
targetFiles = collection.SelectMany(targetFiles, func(subject interface{}) collection.Enumerator {
cast, ok := subject.(string)
if !ok {
return collection.Empty.Enumerate(nil)
}
pathInfo, err := os.Stat(cast)
if err != nil {
return collection.Empty.Enumerate(nil)
}
if pathInfo.IsDir() {
traverser := collection.Directory{
Location: cast,
Options: collection.DirectoryOptionsExcludeDirectories,
}
if useRecursive {
traverser.Options |= collection.DirectoryOptionsRecursive
}
return traverser.Enumerate(nil)
}
return collection.AsEnumerable(cast).Enumerate(nil)
})
targetFiles = collection.Select(targetFiles, func(subject interface{}) interface{} {
return normalizePath(subject.(string))
})
}
// goPath returns the value of the environement variable GOPATH at the beginning of this
// program's execution. The actual enviroment variable is only queried once, before any calls
// are made to this function.
var goPath = func() func() string {
val := normalizePath(os.Getenv("GOPATH"))
return func() string {
return val
}
}()
// getDefaultOutputBase returns the default location of the Azure-SDK-for-Go on your filesystem.
func getDefaultOutputBase() string {
return normalizePath(path.Join(goPath(), "src", "github.com", "Azure", "azure-sdk-for-go"))
}
// trimGoPath operates like strings.TrimPrefix, where the prefix is always the value of GOPATH in the
// environment in which this program is being executed.
func trimGoPath(location string) string {
return strings.TrimPrefix(normalizePath(location), goPath())
}
// normalizePath ensures that a path is expressed using forward slashes for sanity when working
// with Go Standard Library functions that seem to expect this.
func normalizePath(location string) (result string) {
result = strings.Replace(location, `\`, "/", -1)
return
}
// isntNil is a simple `collection.Predicate` which filters out `nil` objects from an Enumerator.
func isntNil(subject interface{}) bool {
return subject != nil
}
@@ -0,0 +1,2 @@
# build products
profileBuilder
@@ -0,0 +1,63 @@
# Profile Builder
## Overview
Azure Profiles offer a means of virtualizing the API Versions of services that should be targeted by an application or SDK.
This concept was introduced for [Azure Stack](https://azure.microsoft.com/overview/azure-stack), where the environment in
which applications will be executed is less consistent than when targeting the public cloud. However, its usefulness as a
means of easily snapping to versions of a service is broadly applicapable. Using profiles, it is easy to use a single version
of models and operations throughout an application, or a means of locking to versions of services that have been tested and
are guaranteed to work together.
[Type aliases were introduced in Go 1.9](https://golang.org/doc/go1.9#language), effectively allowing for multiple symbols
to be mapped to a single type. The impact of this for our support of profiles is tremendous. It allows for seamless
interoperability between packages using different profiles, but where those profiles still target the same API Version of a
service. Without type aliases, we would have been forced to generate code in a way that required some ugly casts to be
scattered throughout the consumer's code.
## Installation
*Note:* These installation notes assume that you have [Go 1.9](https://blog.golang.org/go1.9) or higher, [Glide](http://github.com/Masterminds/glide), and [Git](https://git-scm.com/) installed.
The simplest version of installation is simple, just run the following command:
``` bash
go get -u github.com/Azure/azure-sdk-for-go/tools/profileBuilder
```
If that causes you trouble, run the following commands:
``` bash
# bash
go get -d github.com/Azure/azure-sdk-for-go/tools/profileBuilder
cd $GOPATH/src/github.com/Azure/azure-sdk-for-go/tools/profileBuilder
glide install
go install
```
``` PowerShell
# PowerShell
go get -d github.com/Azure/azure-sdk-for-go/tools/profileBuilder
cd $env:GOPATH\src\github.com\Azure\azure-sdk-for-go\tools\profileBuilder
glide install
go install
```
Taking things a step further, if you'd like the profileBuilder to stamp the version of itself in the code it generates, you can install as below:
``` bash
# bash
go get -d github.com/Azure/azure-sdk-for-go/tools/profileBuilder
cd $GOPATH/src/github.com/Azure/azure-sdk-for-go/tools/profileBuilder
glide install
export currentCommit=$(git rev-parse HEAD)
go install -ldflags "-X main.version=$currentCommit"
```
``` PowerShell
# PowerShell
go get -d github.com/Azure/azure-sdk-for-go/tools/profileBuilder
cd $env:GOPATH\src\github.com\Azure\azure-sdk-for-go\tools\profileBuilder
glide install
$currentCommit = git rev-parse HEAD
go install -ldflags "-X main.version=$currentCommit"
```
+219
View File
@@ -0,0 +1,219 @@
// +build go1.9
// Copyright 2017 Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"fmt"
"io/ioutil"
"log"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"github.com/marstr/collection"
)
// LatestStrategy evaluates the Azure-SDK-for-Go repository for the latest available API versions of each service.
type LatestStrategy struct {
Root string
Predicate func(packageName string) bool
VerboseOutput *log.Logger
}
// AcceptAll is a predefined value for `LatestStrategy.Predicate` which always returns true.
func AcceptAll(name string) bool {
return true
}
// IgnorePreview searches a packages "API Version" to see if it contains the word "preview". It only returns true when a package is not a preview.
func IgnorePreview(name string) (result bool) {
matches := packageName.FindStringSubmatch(name)
version := matches[3]
result = !strings.Contains(version, "-preview") && !strings.Contains(version, "-beta") // matches[2] is the `version` group
return
}
var packageName = regexp.MustCompile(`services[/\\](?P<provider>[\w\-\.\d_\\/]+)[/\\](?:(?P<arm>` + armPathModifier + `)[/\\])?(?P<version>v?\d{4}-\d{2}-\d{2}[\w\d\.\-]*|v?\d+\.\d+[\.\d\w\-]*)[/\\](?P<group>[\w\d\-\._]+)`)
// Enumerate scans through the known Azure SDK for Go packages and finds each
func (latest LatestStrategy) Enumerate(cancel <-chan struct{}) collection.Enumerator {
results := make(chan interface{})
go func() {
defer close(results)
type operationGroup struct {
provider string
arm string
group string
}
type operInfo struct {
version string
rawpath string
}
if latest.VerboseOutput == nil {
latest.VerboseOutput = log.New(ioutil.Discard, "", 0)
}
maxFound := make(map[operationGroup]operInfo)
filepath.Walk(latest.Root, func(currentPath string, info os.FileInfo, openErr error) (err error) {
pathMatches := packageName.FindStringSubmatch(currentPath)
if latest.Predicate == nil {
latest.Predicate = AcceptAll
}
if len(pathMatches) == 0 || !info.IsDir() {
return
} else if !latest.Predicate(currentPath) {
latest.VerboseOutput.Printf("%q rejected by Predicate", currentPath)
return
}
version := pathMatches[3]
currentGroup := operationGroup{
provider: pathMatches[1],
arm: pathMatches[2],
group: pathMatches[4],
}
prev, ok := maxFound[currentGroup]
if !ok {
maxFound[currentGroup] = operInfo{version, currentPath}
latest.VerboseOutput.Printf("New group found %q using version %q", currentGroup, version)
return
}
if le, _ := versionle(prev.version, version); le {
maxFound[currentGroup] = operInfo{version, currentPath}
latest.VerboseOutput.Printf("Updating group %q from version %q to %q", currentGroup, prev.version, version)
} else {
latest.VerboseOutput.Printf("Evaluated group %q version %q decided to stay with %q", currentGroup, version, prev.version)
}
return
})
for _, entry := range maxFound {
absolute, err := filepath.Abs(entry.rawpath)
if err != nil {
continue
}
select {
case results <- absolute:
// Intionally Left Blank
case <-cancel:
return
}
}
}()
return results
}
// ErrNotVersionString is instantiated when a string not conforming to known Azure API Version patterns is parsed is if it did.
type ErrNotVersionString string
func (err ErrNotVersionString) Error() string {
return fmt.Sprintf("`%s` is not a recognized Azure version string", string(err))
}
// versionle takes two version strings that share a format and returns true if the one on the
// left is less than or equal to the one on the right. If the two do not match in format, or
// are not in a well known format, this will return false and an error.
var versionle = func() func(string, string) (bool, error) {
wellKnownStrategies := map[*regexp.Regexp]func([]string, []string) (bool, error){
// The strategy below handles Azure API Versions which have a date optionally followed by some tag.
regexp.MustCompile(`^(?P<year>\d{4})-(?P<month>\d{2})-(?P<day>\d{2})(?:[\.\-](?P<tag>.+))?$`): func(leftMatch, rightMatch []string) (bool, error) {
var err error
for i := 1; i <= 3; i++ { // Start with index 1 because the element 0 is the entire match, not a group. End at 3 because there are three numeric groups.
if leftMatch[i] == rightMatch[i] {
continue
}
var leftNum, rightNum int
leftNum, err = strconv.Atoi(leftMatch[i])
if err != nil {
return false, err
}
rightNum, err = strconv.Atoi(rightMatch[i])
if err != nil {
return false, err
}
if leftNum < rightNum {
return true, nil
}
return false, nil
}
if leftTag, rightTag := leftMatch[4], rightMatch[4]; leftTag == "" && rightTag != "" { // match[4] is the tag portion of a date based API Version label
return false, nil
} else if leftTag != "" && rightTag != "" {
return leftTag <= rightTag, nil
}
return true, nil
},
// The strategy below compares two semvers.
regexp.MustCompile(`(?P<major>\d+)\.(?P<minor>\d+)(?:\.(?P<patch>\d+))?-?(?P<tag>.*)`): func(leftMatch, rightMatch []string) (bool, error) {
for i := 1; i <= 3; i++ {
if len(leftMatch[i]) == 0 || len(rightMatch[i]) == 0 {
return leftMatch[i] <= rightMatch[i], nil
}
numLeft, err := strconv.Atoi(leftMatch[i])
if err != nil {
return false, err
}
numRight, err := strconv.Atoi(rightMatch[i])
if err != nil {
return false, err
}
if numLeft < numRight {
return true, nil
}
if numLeft > numRight {
return false, nil
}
}
return leftMatch[4] <= rightMatch[4], nil
},
}
// This function finds a strategy which recognizes the versions passed to it, then applies that strategy.
return func(left, right string) (bool, error) {
if left == right {
return true, nil
}
for versionStrategy, handler := range wellKnownStrategies {
if leftMatch, rightMatch := versionStrategy.FindAllStringSubmatch(left, 1), versionStrategy.FindAllStringSubmatch(right, 1); len(leftMatch) > 0 && len(rightMatch) > 0 {
return handler(leftMatch[0], rightMatch[0])
}
}
return false, fmt.Errorf("Unable to find versioning strategy that could compare %q and %q", left, right)
}
}()
@@ -0,0 +1,75 @@
// +build go1.9
// Copyright 2017 Microsoft Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"path/filepath"
"testing"
"github.com/marstr/collection"
)
func Test_versionle(t *testing.T) {
const dateWithAlpha, dateWithBeta = "2016-02-01-alpha", "2016-02-01-beta"
const semVer1dot2, semVer1dot3 = "2018-03-03-1.2", "2018-03-03-1.3"
const dateAlone = "2016-12-07"
testCases := []struct {
left string
right string
want bool
}{
{dateWithAlpha, dateWithAlpha, true},
{dateAlone, dateAlone, true},
{"2017-12-01", "2018-03-04", true},
{"2018-03-04", "2017-12-01", false},
{semVer1dot2, semVer1dot3, true},
{semVer1dot3, semVer1dot2, false},
{semVer1dot2, semVer1dot2, true},
{dateWithAlpha, dateWithBeta, true},
{dateWithBeta, dateWithAlpha, false},
{"2016-04-03-preview", "2016-04-03", true},
{"2016-04-03", "2016-04-03-preview", false},
{"1.0.0", "5.6", true},
{"5.6", "1.0.0", false},
{"5.6", "6.0", true},
{"6.0", "5.6", false},
}
for _, tc := range testCases {
t.Run("", func(t *testing.T) {
if got, err := versionle(tc.left, tc.right); err != nil {
t.Error(err)
} else if got != tc.want {
t.Logf("\n Left: %s\nRight: %s", tc.left, tc.right)
t.Logf("\n got: %v\nwant: %v", got, tc.want)
t.Fail()
}
})
}
}
func BenchmarkLatestStrategy_Enumerate(b *testing.B) {
subject := LatestStrategy{
Root: filepath.Join("..", "..", "service"),
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
b.Logf("Enumerated %d packages", collection.CountAll(subject))
}
}
+60
View File
@@ -0,0 +1,60 @@
// +build go1.9
// Copyright 2017 Microsoft Corporation and contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"fmt"
"io"
"os"
"path"
"github.com/marstr/collection"
)
// ListStrategy allows a mechanism for a list of packages that should be included in a profile.
type ListStrategy struct {
io.Reader
}
// Enumerate reads a new line delimited list of packages names relative to $GOPATH
func (list ListStrategy) Enumerate(cancel <-chan struct{}) collection.Enumerator {
results := make(chan interface{})
go func() {
defer close(results)
var currentLine string
for {
_, err := fmt.Fscanln(list, &currentLine)
if err != nil {
return
}
currentLine = path.Join(os.Getenv("GOPATH"), "src", currentLine)
select {
case results <- currentLine:
// Intentionally Left Blank
case <-cancel:
return
}
}
}()
return results
}
@@ -0,0 +1,86 @@
// +build go1.9
// Copyright 2017 Microsoft Corporation and contributors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"bytes"
"io"
"os"
"path"
"path/filepath"
"reflect"
"strings"
"testing"
)
func TestList_Enumerate(t *testing.T) {
smallProfile, err := os.Open(filepath.Join("testdata", "smallProfile.txt"))
if err != nil {
t.Error(err)
return
}
gopath := strings.Replace(os.Getenv("GOPATH"), "\\", "/", -1)
testCases := []struct {
io.Reader
expected map[string]struct{}
}{
{
bytes.NewReader([]byte("a.md")),
map[string]struct{}{path.Join(gopath, "src", "a.md"): struct{}{}},
},
{
smallProfile,
map[string]struct{}{
path.Join(gopath, "src", "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2015-06-15/compute"): struct{}{},
path.Join(gopath, "src", "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2015-06-15/network"): struct{}{},
},
},
}
for _, tc := range testCases {
subject := ListStrategy{Reader: tc}
t.Run("", func(t *testing.T) {
done := make(chan struct{})
defer close(done)
for currentLine := range subject.Enumerate(done) {
cast, ok := currentLine.(string)
if !ok {
t.Logf("ListStrategy returned a %q instead of a \"string\"", reflect.TypeOf(currentLine).Name())
t.Fail()
return
}
cast = strings.Replace(cast, "\\", "/", -1)
if _, ok = tc.expected[cast]; !ok {
t.Logf("Unexpected value %q encountered", cast)
t.Fail()
} else {
delete(tc.expected, cast)
}
}
for unseen := range tc.expected {
t.Logf("Expected value %q was unencountered", unseen)
t.Fail()
}
})
}
}
@@ -0,0 +1,397 @@
// +build go1.9
// Copyright 2017 Microsoft Corporation and contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// profileBuilder creates a series of packages filled entirely with alias types
// and functions supporting those alias types by directing traffic to the
// functions supporting the original types. This is useful associating a series
// of packages in separate API Versions for easier/safer use.
//
// The Azure-SDK-for-Go teams intends to use this tool to generated profiles
// that we will publish in this repository for general use. However, this tool
// in the case that one has their own list of Services at given API Versions,
// this may prove to be a useful tool for you.
package main
import (
"bytes"
"errors"
"flag"
"fmt"
"go/ast"
"go/parser"
"go/printer"
"go/token"
"io"
"io/ioutil"
"log"
"os"
"os/exec"
"path"
"path/filepath"
"strings"
"time"
"github.com/marstr/collection"
goalias "github.com/marstr/goalias/model"
"github.com/marstr/randname"
)
var (
profileName string
outputLocation string
inputRoot string
inputList io.Reader
packageStrategy collection.Enumerable
outputLog *log.Logger
errLog *log.Logger
)
// WellKnownStrategy is an Enumerable which lists all known strategies for choosing packages for a profile.
type WellKnownStrategy string
// This block declares the definitive list of WellKnownStrategies
const (
WellKnownStrategyList WellKnownStrategy = "list"
WellKnownStrategyLatest WellKnownStrategy = "latest"
WellKnownStrategyPreview WellKnownStrategy = "preview"
)
const armPathModifier = "mgmt"
// If not the empty string, this string should be stamped into files generated by the profileBuilder.
// Note: This variable should be set by passing the argument "-X main.version=`{your value}`" to the Go linker. example: `go build -ldflags "-X main.version=f43d726b6e3f1e3eb7cbdba3982f0253000d5dc5"`
var version string
func main() {
var packages collection.Enumerator
type alias struct {
*goalias.AliasPackage
TargetPath string
}
// Find the names of all of the packages for inclusion in this profile.
packages = packageStrategy.Enumerate(nil).Select(func(x interface{}) interface{} {
if cast, ok := x.(string); ok {
return cast
}
return nil
})
// Parse the packages that were selected for inclusion in this profile.
packages = packages.SelectMany(func(x interface{}) collection.Enumerator {
results := make(chan interface{})
go func() {
defer close(results)
cast, ok := x.(string)
if !ok {
return
}
files := token.NewFileSet()
parsed, err := parser.ParseDir(files, cast, nil, 0)
if err != nil {
errLog.Printf("Couldn't open %q because: %v", cast, err)
return
}
for _, entry := range parsed {
results <- entry
}
}()
return results
})
// Generate the alias package from the originally parsed one.
packages = packages.ParallelSelect(func(x interface{}) interface{} {
var err error
var subject *goalias.AliasPackage
cast, ok := x.(*ast.Package)
if !ok {
return nil
}
var bundle alias
for filename := range cast.Files {
bundle.TargetPath = filepath.Dir(filename)
bundle.TargetPath = trimGoPath(bundle.TargetPath)
subject, err = goalias.NewAliasPackage(cast, bundle.TargetPath)
if err != nil {
errLog.Print(err)
return nil
}
bundle.TargetPath, err = getAliasPath(bundle.TargetPath, profileName)
if err != nil {
errLog.Print(err)
return nil
}
break
}
bundle.AliasPackage = subject
return &bundle
})
packages = packages.Where(func(x interface{}) bool {
return x != nil
})
// Update the "UserAgent" function in the generated profile, if it is present.
packages = packages.Select(func(x interface{}) interface{} {
cast := x.(*alias)
var userAgent *ast.FuncDecl
// Grab all functions in the alias package named "UserAgent"
userAgentCandidates := collection.Where(collection.AsEnumerable(cast.Files["models.go"].Decls), func(x interface{}) bool {
cast, ok := x.(*ast.FuncDecl)
return ok && cast.Name.Name == "UserAgent"
})
// There should really only be one of them, otherwise bailout because we don't understand the world anymore.
candidate, err := collection.Single(userAgentCandidates)
if err != nil {
return x
}
userAgent, ok := candidate.(*ast.FuncDecl)
if !ok {
return x
}
// Grab the expression being returned.
retResults := &userAgent.Body.List[0].(*ast.ReturnStmt).Results[0]
// Append a string literal to the result
updated := &ast.BinaryExpr{
Op: token.ADD,
X: *retResults,
Y: &ast.BasicLit{
Value: fmt.Sprintf("\" profiles/%s\"", profileName),
},
}
*retResults = updated
return x
})
// Add the MSFT Copyright Header, then write the alias package to disk.
products := packages.ParallelSelect(func(x interface{}) interface{} {
cast, ok := x.(*alias)
if !ok {
return false
}
files := token.NewFileSet()
outputPath := filepath.Join(outputLocation, cast.TargetPath, "models.go")
outputPath = strings.Replace(outputPath, `\`, `/`, -1)
err := os.MkdirAll(path.Dir(outputPath), os.ModePerm|os.ModeDir)
if err != nil {
errLog.Print("error creating directory:", err)
return false
}
outputFile, err := os.Create(outputPath)
if err != nil {
errLog.Print("error creating file: ", err)
return false
}
// TODO: This should really be added by the `goalias` package itself. Doing it here is a work around
fmt.Fprintln(outputFile, "// +build go1.9")
fmt.Fprintln(outputFile)
generatorStampBuilder := new(bytes.Buffer)
fmt.Fprintf(generatorStampBuilder, "// Copyright %4d Microsoft Corporation\n", time.Now().Year())
fmt.Fprintln(generatorStampBuilder, `//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.`)
fmt.Fprintln(outputFile, generatorStampBuilder.String())
generatorStampBuilder.Reset()
fmt.Fprintln(generatorStampBuilder, "// This code was auto-generated by:")
fmt.Fprintln(generatorStampBuilder, "// github.com/Azure/azure-sdk-for-go/tools/profileBuilder")
if version != "" {
fmt.Fprintln(generatorStampBuilder, "// commit ID:", version)
}
fmt.Fprintln(generatorStampBuilder)
fmt.Fprint(outputFile, generatorStampBuilder.String())
outputLog.Printf("Writing File: %s", outputPath)
printer.Fprint(outputFile, files, cast.ModelFile())
return true
})
generated := 0
// Write each aliased package that was found
for entry := range products {
if entry.(bool) {
generated++
}
}
outputLog.Print(generated, " packages generated.")
if err := exec.Command("gofmt", "-w", outputLocation).Run(); err == nil {
outputLog.Print("Success formatting profile.")
} else {
errLog.Print("Trouble formatting profile: ", err)
}
}
func init() {
const defaultName = "{randomly generated}"
var selectedStrategy string
var inputListLocation string
var useVerbose bool
flag.StringVar(&profileName, "name", defaultName, "The name that should be given to the generated profile.")
flag.StringVar(&outputLocation, "o", defaultOutputLocation(), "The output location for the package generated as a profile.")
flag.StringVar(&inputRoot, "root", defaultInputRoot(), "The location of the Azure SDK for Go's service packages.")
flag.StringVar(&inputListLocation, "l", "", "If the `list` strategy is chosen, -l is the location of the file to read for said list. If not present, stdin is used.")
flag.StringVar(&selectedStrategy, "s", string(WellKnownStrategyLatest), "The strategy to employ for finding packages to put in a profile.")
flag.BoolVar(&useVerbose, "v", false, "Write status to stderr as the program progresses")
flag.Parse()
// Setup Verbose Status Log and Error Log
var logWriter io.Writer
if useVerbose {
logWriter = os.Stderr
} else {
logWriter = ioutil.Discard
}
outputLog = log.New(logWriter, "[STATUS] ", 0)
outputLog.Print("Status Logging Enabled")
errLog = log.New(logWriter, "[ERROR] ", 0)
if version != "" {
outputLog.Print("profileBuilder Version: ", version)
}
// Sort out the Profile Name to be used.
if profileName == defaultName {
profileName = randname.AdjNoun{}.Generate()
outputLog.Print("Profile Name Set to: ", profileName)
}
inputList = os.Stdin
if inputListLocation == "" {
outputLog.Print("Reading input from standard input")
} else {
var err error
outputLog.Print("Reading input from: ", inputListLocation)
inputList, err = os.Open(inputListLocation)
if err != nil {
errLog.Print(err)
os.Exit(1)
}
}
wellKnownStrategies := map[WellKnownStrategy]collection.Enumerable{
WellKnownStrategyList: ListStrategy{Reader: inputList},
WellKnownStrategyLatest: LatestStrategy{Root: inputRoot, Predicate: IgnorePreview, VerboseOutput: outputLog},
WellKnownStrategyPreview: LatestStrategy{Root: inputRoot, Predicate: AcceptAll},
}
if s, ok := wellKnownStrategies[WellKnownStrategy(selectedStrategy)]; ok {
packageStrategy = s
outputLog.Printf("Using Well Known Strategy: %s", selectedStrategy)
} else {
errLog.Printf("Unknown strategy for identifying packages: %s\n", selectedStrategy)
os.Exit(1)
}
}
// AzureSDKforGoLocation returns the default location for the Azure-SDK-for-Go to reside.
func AzureSDKforGoLocation() string {
return path.Join(
os.Getenv("GOPATH"),
"src",
"github.com",
"Azure",
"azure-sdk-for-go",
)
}
func defaultOutputLocation() string {
return path.Join(AzureSDKforGoLocation(), "profiles")
}
func defaultInputRoot() string {
return path.Join(AzureSDKforGoLocation(), "services")
}
// getAliasPath takes an existing API Version path and a package name, and converts the path
// to a path which uses the new profile layout.
func getAliasPath(subject, profile string) (transformed string, err error) {
subject = strings.TrimSuffix(subject, "/")
subject = trimGoPath(subject)
matches := packageName.FindAllStringSubmatch(subject, -1)
if matches == nil {
err = errors.New("path does not resemble a known package path")
return
}
output := []string{
profile,
matches[0][1],
}
if matches[0][2] == armPathModifier {
output = append(output, armPathModifier)
}
output = append(output, matches[0][4])
transformed = strings.Join(output, "/")
return
}
// trimGoPath removes the prefix defined in the environment variabe GOPATH if it is present in the string provided.
var trimGoPath = func() func(string) string {
splitGo := strings.Split(os.Getenv("GOPATH"), string(os.PathSeparator))
splitGo = append(splitGo, "src")
return func(subject string) string {
splitPath := strings.Split(subject, string(os.PathSeparator))
for i, dir := range splitGo {
if splitPath[i] != dir {
return subject
}
}
packageIdentifier := splitPath[len(splitGo):]
return path.Join(packageIdentifier...)
}
}()
@@ -0,0 +1,94 @@
// +build go1.9
// Copyright 2017 Microsoft Corporation and contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// profileBuilder creates a series of packages filled entirely with alias types
// and functions supporting those alias types by directing traffic to the
// functions supporting the original types. This is useful associating a series
// of packages in separate API Versions for easier/safer use.
//
// The Azure-SDK-for-Go teams intends to use this tool to generated profiles
// that we will publish in this repository for general use. However, this tool
// in the case that one has their own list of Services at given API Versions,
// this may prove to be a useful tool for you.
package main
import (
"path/filepath"
"strings"
"testing"
)
func Test_getAliasPath(t *testing.T) {
const profileName = "profile1"
testCases := []struct {
original string
expected string
}{
{
filepath.Join("services", "cdn", "mgmt", "2015-06-01", "cdn"),
filepath.Join(profileName, "cdn", "mgmt", "cdn"),
},
{
filepath.Join("services", "keyvault", "2016-10-01", "keyvault"),
filepath.Join(profileName, "keyvault", "keyvault"),
},
{
filepath.Join("services", "keyvault", "mgmt", "2016-10-01", "keyvault"),
filepath.Join(profileName, "keyvault", "mgmt", "keyvault"),
},
{
filepath.Join("services", "datalake", "analytics", "2016-11-01-preview", "catalog"),
filepath.Join(profileName, "datalake", "analytics", "catalog"),
},
}
const consistentSeperator = "/"
pathNorm := func(location string) string {
return strings.Replace(location, `\`, consistentSeperator, -1)
}
pathsEqual := func(left, right string) bool {
left, right = pathNorm(left), pathNorm(right)
pieceWiseLeft, pieceWiseRight := strings.Split(left, consistentSeperator), strings.Split(right, consistentSeperator)
if len(pieceWiseLeft) != len(pieceWiseRight) {
return false
}
for i, lval := range pieceWiseLeft {
rval := pieceWiseRight[i]
if lval != rval {
return false
}
}
return true
}
for _, tc := range testCases {
t.Run(tc.original, func(t *testing.T) {
got, err := getAliasPath(tc.original, profileName)
if err != nil {
t.Error(err)
}
if !pathsEqual(tc.expected, got) {
t.Logf("\ngot: \t%q\nwant:\t%q", pathNorm(got), pathNorm(tc.expected))
t.Fail()
}
})
}
}
@@ -0,0 +1,5 @@
#!bin/bash
set -ev
if [[ ! "${TRAVIS_GO_VERSION}" < "1.9" ]]; then
go test -v github.com/Azure/azure-sdk-for-go/tools/profileBuilder
fi
@@ -0,0 +1,2 @@
github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2015-06-15/compute
github.com/Azure/azure-sdk-for-go/services/network/mgmt/2015-06-15/network