diff --git a/.gitignore b/.gitignore index 52e8dc88..8e61eeca 100644 --- a/.gitignore +++ b/.gitignore @@ -30,5 +30,8 @@ bin # build artifacts _out -# generated -release/kustomization.yaml + +# test binaries and kubeconfigs +argo +kubemcsa +kubeconfig-cluster* \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..3af31dd6 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,45 @@ +# Changelog + + + +## v0.4.0 + +### New Features + +- Helm charts (v3) for an easier and more flexible installation +- Multi-federation that works! +- Better RBAC with cluster namespaces: as an option, you can setup multicluster-scheduler so that each member cluster has a dedicated namespace in the scheduler cluster for observations and decisions. This makes it possible for partially trusted clusters to participate in the same federation (they can send pods to one another, via the scheduler, but they cannot observe one another). +- More observations (to support non-basic schedulers, including [Admiralty's advanced scheduler]()) + +### Bugfixes + +- Don't spam the log with update errors like "the object has been modified; please apply your changes to the latest version and try again". The controller would back off and retry, so the log was confusing. We now just ignore the error and let the cache enqueue a reconcile request when it receives the latest version (no back-off, but for that, the controller must watch the updated resource). + +### Internals + +- Use `gc` pattern from [multicluster-controller](https://github.com/admiraltyio/multicluster-controller) for cross-cluster/cross-namespace garbage collection with finalizers for `send`, `receive`, and `bind` controllers. (As a result, the local `EnqueueRequestForMulticlusterController` handler in `receive` was deleted. It now exists in multicluster-controller.) +- Split `bind` controller from `schedule` controller, to more easily plug in custom schedulers. +- `send` controller uses `unstructured` to support more observations. +- Switch to Go modules +- Faster end-to-end tests with [KIND (Kubernetes in Docker)](https://kind.sigs.k8s.io/) rather than GKE. +- Stop using skaffold (build images once in `build.sh`) and kustomize (because we now use Helm). +- Split `delegatestate` controller (in scheduler manager) from `feedback` controller (in agent manager) to make cluster namespace feature possible (where cluster1 cannot see observations from cluster2). +- The source cluster name of an observation is either its namespace (in cluster namespace mode) or the cross-cluster GC label `multicluster.admiralty.io/parent-clusterName`. The target cluster name of a decision is either its namespace (in cluster namespace mode) or the `multicluster.admiralty.io/clustername` annotation (added by `bind` and `globalsvc`). `status.liveState.metadata.ClusterName` is not longer used, except when it's backfilled in memory at the interface with the basic scheduler, which still uses the field internally. diff --git a/Gopkg.lock b/Gopkg.lock deleted file mode 100644 index f7b72dbe..00000000 --- a/Gopkg.lock +++ /dev/null @@ -1,989 +0,0 @@ -# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. - - -[[projects]] - digest = "1:b02ee004bf5af0be6c8cd94ba78e1dc6da62af42f6b0eba1110993f91c3f7cdb" - name = "admiralty.io/multicluster-controller" - packages = [ - "pkg/cluster", - "pkg/controller", - "pkg/handler", - "pkg/manager", - "pkg/reconcile", - "pkg/reference", - ] - pruneopts = "UT" - revision = "93617a36af06b35194817cb953fc344f42290bd4" - version = "v0.1.0" - -[[projects]] - digest = "1:0266af8eb973d89bd8dbfde04f4a184b798a905ff28d7758bcbfdeb9b264bbfb" - name = "admiralty.io/multicluster-service-account" - packages = ["pkg/config"] - pruneopts = "UT" - revision = "1e95f30ec1613af35f4e7abd0fa0c9bd31e00daf" - version = "v0.2.0" - -[[projects]] - branch = "master" - digest = "1:d6afaeed1502aa28e80a4ed0981d570ad91b2579193404256ce672ed0a609e0d" - name = "github.com/beorn7/perks" - packages = ["quantile"] - pruneopts = "UT" - revision = "3a771d992973f24aa725d07868b467d1ddfceafb" - -[[projects]] - digest = "1:ffe9824d294da03b391f44e1ae8281281b4afc1bdaa9588c9097785e3af10cec" - name = "github.com/davecgh/go-spew" - packages = ["spew"] - pruneopts = "UT" - revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73" - version = "v1.1.1" - -[[projects]] - branch = "master" - digest = "1:ecdc8e0fe3bc7d549af1c9c36acf3820523b707d6c071b6d0c3860882c6f7b42" - name = "github.com/docker/spdystream" - packages = [ - ".", - "spdy", - ] - pruneopts = "UT" - revision = "6480d4af844c189cf5dd913db24ddd339d3a4f85" - -[[projects]] - digest = "1:2cd7915ab26ede7d95b8749e6b1f933f1c6d5398030684e6505940a10f31cfda" - name = "github.com/ghodss/yaml" - packages = ["."] - pruneopts = "UT" - revision = "0ca9ea5df5451ffdf184b4428c902747c2c11cd7" - version = "v1.0.0" - -[[projects]] - branch = "master" - digest = "1:edd2fa4578eb086265db78a9201d15e76b298dfd0d5c379da83e9c61712cf6df" - name = "github.com/go-logr/logr" - packages = ["."] - pruneopts = "UT" - revision = "9fb12b3b21c5415d16ac18dc5cd42c1cfdd40c4e" - -[[projects]] - digest = "1:ce43ad4015e7cdad3f0e8f2c8339439dd4470859a828d2a6988b0f713699e94a" - name = "github.com/go-logr/zapr" - packages = ["."] - pruneopts = "UT" - revision = "7536572e8d55209135cd5e7ccf7fce43dca217ab" - version = "v0.1.0" - -[[projects]] - digest = "1:7f89e0c888fb99c61055c646f5678aae645b0b0a1443d9b2dcd9964d850827ce" - name = "github.com/go-test/deep" - packages = ["."] - pruneopts = "UT" - revision = "6592d9cc0a499ad2d5f574fde80a2b5c5cc3b4f5" - version = "v1.0.1" - -[[projects]] - digest = "1:bd342004ef1bb83bee8709e2ad630e47a95d069f7fcb60ee1b8bceb4bbbe14ae" - name = "github.com/gobuffalo/envy" - packages = ["."] - pruneopts = "UT" - revision = "801d7253ade1f895f74596b9a96147ed2d3b087e" - version = "v1.6.11" - -[[projects]] - digest = "1:b402bb9a24d108a9405a6f34675091b036c8b056aac843bf6ef2389a65c5cf48" - name = "github.com/gogo/protobuf" - packages = [ - "proto", - "sortkeys", - ] - pruneopts = "UT" - revision = "4cbf7e384e768b4e01799441fdf2a706a5635ae7" - version = "v1.2.0" - -[[projects]] - branch = "master" - digest = "1:1ba1d79f2810270045c328ae5d674321db34e3aae468eb4233883b473c5c0467" - name = "github.com/golang/glog" - packages = ["."] - pruneopts = "UT" - revision = "23def4e6c14b4da8ac2ed8007337bc5eb5007998" - -[[projects]] - branch = "master" - digest = "1:3fb07f8e222402962fa190eb060608b34eddfb64562a18e2167df2de0ece85d8" - name = "github.com/golang/groupcache" - packages = ["lru"] - pruneopts = "UT" - revision = "c65c006176ff7ff98bb916961c7abbc6b0afc0aa" - -[[projects]] - digest = "1:4c0989ca0bcd10799064318923b9bc2db6b4d6338dd75f3f2d86c3511aaaf5cf" - name = "github.com/golang/protobuf" - packages = [ - "proto", - "ptypes", - "ptypes/any", - "ptypes/duration", - "ptypes/timestamp", - ] - pruneopts = "UT" - revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5" - version = "v1.2.0" - -[[projects]] - branch = "master" - digest = "1:0bfbe13936953a98ae3cfe8ed6670d396ad81edf069a806d2f6515d7bb6950df" - name = "github.com/google/btree" - packages = ["."] - pruneopts = "UT" - revision = "4030bb1f1f0c35b30ca7009e9ebd06849dd45306" - -[[projects]] - branch = "master" - digest = "1:3ee90c0d94da31b442dde97c99635aaafec68d0b8a3c12ee2075c6bdabeec6bb" - name = "github.com/google/gofuzz" - packages = ["."] - pruneopts = "UT" - revision = "24818f796faf91cd76ec7bddd72458fbced7a6c1" - -[[projects]] - digest = "1:236d7e1bdb50d8f68559af37dbcf9d142d56b431c9b2176d41e2a009b664cda8" - name = "github.com/google/uuid" - packages = ["."] - pruneopts = "UT" - revision = "9b3b1e0f5f99ae461456d768e7d301a7acdaa2d8" - version = "v1.1.0" - -[[projects]] - digest = "1:65c4414eeb350c47b8de71110150d0ea8a281835b1f386eacaa3ad7325929c21" - name = "github.com/googleapis/gnostic" - packages = [ - "OpenAPIv2", - "compiler", - "extensions", - ] - pruneopts = "UT" - revision = "7c663266750e7d82587642f65e60bc4083f1f84e" - version = "v0.2.0" - -[[projects]] - branch = "master" - digest = "1:86c1210529e69d69860f2bb3ee9ccce0b595aa3f9165e7dd1388e5c612915888" - name = "github.com/gregjones/httpcache" - packages = [ - ".", - "diskcache", - ] - pruneopts = "UT" - revision = "c63ab54fda8f77302f8d414e19933f2b6026a089" - -[[projects]] - digest = "1:8ec8d88c248041a6df5f6574b87bc00e7e0b493881dad2e7ef47b11dc69093b5" - name = "github.com/hashicorp/golang-lru" - packages = [ - ".", - "simplelru", - ] - pruneopts = "UT" - revision = "20f1fb78b0740ba8c3cb143a61e86ba5c8669768" - version = "v0.5.0" - -[[projects]] - digest = "1:a1038ef593beb4771c8f0f9c26e8b00410acd800af5c6864651d9bf160ea1813" - name = "github.com/hpcloud/tail" - packages = [ - ".", - "ratelimiter", - "util", - "watch", - "winfile", - ] - pruneopts = "UT" - revision = "a30252cb686a21eb2d0b98132633053ec2f7f1e5" - version = "v1.0.0" - -[[projects]] - digest = "1:8eb1de8112c9924d59bf1d3e5c26f5eaa2bfc2a5fcbb92dc1c2e4546d695f277" - name = "github.com/imdario/mergo" - packages = ["."] - pruneopts = "UT" - revision = "9f23e2d6bd2a77f959b2bf6acdbefd708a83a4a4" - version = "v0.3.6" - -[[projects]] - digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be" - name = "github.com/inconshreveable/mousetrap" - packages = ["."] - pruneopts = "UT" - revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75" - version = "v1.0" - -[[projects]] - digest = "1:ecd9aa82687cf31d1585d4ac61d0ba180e42e8a6182b85bd785fcca8dfeefc1b" - name = "github.com/joho/godotenv" - packages = ["."] - pruneopts = "UT" - revision = "23d116af351c84513e1946b527c88823e476be13" - version = "v1.3.0" - -[[projects]] - digest = "1:3e551bbb3a7c0ab2a2bf4660e7fcad16db089fdcfbb44b0199e62838038623ea" - name = "github.com/json-iterator/go" - packages = ["."] - pruneopts = "UT" - revision = "1624edc4454b8682399def8740d46db5e4362ba4" - version = "v1.1.5" - -[[projects]] - digest = "1:3804a3a02964db8e6db3e5e7960ac1c1a9b12835642dd4f4ac4e56c749ec73eb" - name = "github.com/markbates/inflect" - packages = ["."] - pruneopts = "UT" - revision = "24b83195037b3bc61fcda2d28b7b0518bce293b6" - version = "v1.0.4" - -[[projects]] - digest = "1:464aef731a5f82ded547c62e249a2e9ec59fbbc9ddab53cda7b9857852630a61" - name = "github.com/mattbaird/jsonpatch" - packages = ["."] - pruneopts = "UT" - revision = "7c0e3b262f30165a8ec3d0b4c6059fd92703bfb2" - source = "https://github.com/appscode/jsonpatch.git" - version = "1.0.0" - -[[projects]] - digest = "1:ff5ebae34cfbf047d505ee150de27e60570e8c394b3b8fdbb720ff6ac71985fc" - name = "github.com/matttproud/golang_protobuf_extensions" - packages = ["pbutil"] - pruneopts = "UT" - revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c" - version = "v1.0.1" - -[[projects]] - digest = "1:33422d238f147d247752996a26574ac48dcf472976eda7f5134015f06bf16563" - name = "github.com/modern-go/concurrent" - packages = ["."] - pruneopts = "UT" - revision = "bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94" - version = "1.0.3" - -[[projects]] - digest = "1:e32bdbdb7c377a07a9a46378290059822efdce5c8d96fe71940d87cb4f918855" - name = "github.com/modern-go/reflect2" - packages = ["."] - pruneopts = "UT" - revision = "4b7aa43c6742a2c18fdef89dd197aaae7dac7ccd" - version = "1.0.1" - -[[projects]] - digest = "1:5f4b78246f0bcb105b1e3b2b9e22b52a57cd02f57a8078572fe27c62f4a75ff7" - name = "github.com/onsi/ginkgo" - packages = [ - ".", - "config", - "internal/codelocation", - "internal/containernode", - "internal/failer", - "internal/leafnodes", - "internal/remote", - "internal/spec", - "internal/spec_iterator", - "internal/specrunner", - "internal/suite", - "internal/testingtproxy", - "internal/writer", - "reporters", - "reporters/stenographer", - "reporters/stenographer/support/go-colorable", - "reporters/stenographer/support/go-isatty", - "types", - ] - pruneopts = "UT" - revision = "2e1be8f7d90e9d3e3e58b0ce470f2f14d075406f" - version = "v1.7.0" - -[[projects]] - digest = "1:b4764603c54d74435f246901248aefb2b9d430bb7b160afde1afc41d89d48f1a" - name = "github.com/onsi/gomega" - packages = [ - ".", - "format", - "gbytes", - "gexec", - "internal/assertion", - "internal/asyncassertion", - "internal/oraclematcher", - "internal/testingtsupport", - "matchers", - "matchers/support/goraph/bipartitegraph", - "matchers/support/goraph/edge", - "matchers/support/goraph/node", - "matchers/support/goraph/util", - "types", - ] - pruneopts = "UT" - revision = "65fb64232476ad9046e57c26cd0bff3d3a8dc6cd" - version = "v1.4.3" - -[[projects]] - digest = "1:e5d0bd87abc2781d14e274807a470acd180f0499f8bf5bb18606e9ec22ad9de9" - name = "github.com/pborman/uuid" - packages = ["."] - pruneopts = "UT" - revision = "adf5a7427709b9deb95d29d3fa8a2bf9cfd388f1" - version = "v1.2" - -[[projects]] - branch = "master" - digest = "1:3bf17a6e6eaa6ad24152148a631d18662f7212e21637c2699bff3369b7f00fa2" - name = "github.com/petar/GoLLRB" - packages = ["llrb"] - pruneopts = "UT" - revision = "53be0d36a84c2a886ca057d34b6aa4468df9ccb4" - -[[projects]] - digest = "1:0e7775ebbcf00d8dd28ac663614af924411c868dca3d5aa762af0fae3808d852" - name = "github.com/peterbourgon/diskv" - packages = ["."] - pruneopts = "UT" - revision = "5f041e8faa004a95c88a202771f4cc3e991971e6" - version = "v2.0.1" - -[[projects]] - digest = "1:cf31692c14422fa27c83a05292eb5cbe0fb2775972e8f1f8446a71549bd8980b" - name = "github.com/pkg/errors" - packages = ["."] - pruneopts = "UT" - revision = "ba968bfe8b2f7e042a574c888954fccecfa385b4" - version = "v0.8.1" - -[[projects]] - digest = "1:93a746f1060a8acbcf69344862b2ceced80f854170e1caae089b2834c5fbf7f4" - name = "github.com/prometheus/client_golang" - packages = [ - "prometheus", - "prometheus/internal", - "prometheus/promhttp", - ] - pruneopts = "UT" - revision = "505eaef017263e299324067d40ca2c48f6a2cf50" - version = "v0.9.2" - -[[projects]] - branch = "master" - digest = "1:2d5cd61daa5565187e1d96bae64dbbc6080dacf741448e9629c64fd93203b0d4" - name = "github.com/prometheus/client_model" - packages = ["go"] - pruneopts = "UT" - revision = "f287a105a20ec685d797f65cd0ce8fbeaef42da1" - -[[projects]] - branch = "master" - digest = "1:ce62b400185bf6b16ef6088011b719e449f5c15c4adb6821589679f752c2788e" - name = "github.com/prometheus/common" - packages = [ - "expfmt", - "internal/bitbucket.org/ww/goautoneg", - "model", - ] - pruneopts = "UT" - revision = "2998b132700a7d019ff618c06a234b47c1f3f681" - -[[projects]] - branch = "master" - digest = "1:08eb8b60450efe841e37512d66ce366a87d187505d7c67b99307a6c1803483a2" - name = "github.com/prometheus/procfs" - packages = [ - ".", - "internal/util", - "nfs", - "xfs", - ] - pruneopts = "UT" - revision = "b1a0a9a36d7453ba0f62578b99712f3a6c5f82d1" - -[[projects]] - digest = "1:efafa5160b1666d27e6e1ad4cd3fba27ce73aaaa1e99c3d02e14e55690a8878d" - name = "github.com/rogpeppe/go-internal" - packages = [ - "modfile", - "module", - "semver", - ] - pruneopts = "UT" - revision = "d87f08a7d80821c797ffc8eb8f4e01675f378736" - version = "v1.0.0" - -[[projects]] - digest = "1:d707dbc1330c0ed177d4642d6ae102d5e2c847ebd0eb84562d0dc4f024531cfc" - name = "github.com/spf13/afero" - packages = [ - ".", - "mem", - ] - pruneopts = "UT" - revision = "a5d6946387efe7d64d09dcba68cdd523dc1273a3" - version = "v1.2.0" - -[[projects]] - digest = "1:645cabccbb4fa8aab25a956cbcbdf6a6845ca736b2c64e197ca7cbb9d210b939" - name = "github.com/spf13/cobra" - packages = ["."] - pruneopts = "UT" - revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385" - version = "v0.0.3" - -[[projects]] - digest = "1:c1b1102241e7f645bc8e0c22ae352e8f0dc6484b6cb4d132fa9f24174e0119e2" - name = "github.com/spf13/pflag" - packages = ["."] - pruneopts = "UT" - revision = "298182f68c66c05229eb03ac171abe6e309ee79a" - version = "v1.0.3" - -[[projects]] - digest = "1:3c1a69cdae3501bf75e76d0d86dc6f2b0a7421bc205c0cb7b96b19eed464a34d" - name = "go.uber.org/atomic" - packages = ["."] - pruneopts = "UT" - revision = "1ea20fb1cbb1cc08cbd0d913a96dead89aa18289" - version = "v1.3.2" - -[[projects]] - digest = "1:60bf2a5e347af463c42ed31a493d817f8a72f102543060ed992754e689805d1a" - name = "go.uber.org/multierr" - packages = ["."] - pruneopts = "UT" - revision = "3c4937480c32f4c13a875a1829af76c98ca3d40a" - version = "v1.1.0" - -[[projects]] - digest = "1:c52caf7bd44f92e54627a31b85baf06a68333a196b3d8d241480a774733dcf8b" - name = "go.uber.org/zap" - packages = [ - ".", - "buffer", - "internal/bufferpool", - "internal/color", - "internal/exit", - "zapcore", - ] - pruneopts = "UT" - revision = "ff33455a0e382e8a81d14dd7c922020b6b5e7982" - version = "v1.9.1" - -[[projects]] - branch = "master" - digest = "1:38f553aff0273ad6f367cb0a0f8b6eecbaef8dc6cb8b50e57b6a81c1d5b1e332" - name = "golang.org/x/crypto" - packages = ["ssh/terminal"] - pruneopts = "UT" - revision = "ff983b9c42bc9fbf91556e191cc8efb585c16908" - -[[projects]] - branch = "master" - digest = "1:2217740ae418ff26c5b916946de6ab741b7267bb9b126cc8ddd1375b34b5b268" - name = "golang.org/x/net" - packages = [ - "context", - "context/ctxhttp", - "html", - "html/atom", - "html/charset", - "http/httpguts", - "http2", - "http2/hpack", - "idna", - ] - pruneopts = "UT" - revision = "915654e7eabcea33ae277abbecf52f0d8b7a9fdc" - -[[projects]] - branch = "master" - digest = "1:0977fc80383ba48e634c807e8e47d4ceab54759944587c2233bf3c624bed91cc" - name = "golang.org/x/oauth2" - packages = [ - ".", - "internal", - ] - pruneopts = "UT" - revision = "fd3eaa146cbb5c89ce187c275fb79bd3a36a5ffc" - -[[projects]] - branch = "master" - digest = "1:5ee4df7ab18e945607ac822de8d10b180baea263b5e8676a1041727543b9c1e4" - name = "golang.org/x/sys" - packages = [ - "unix", - "windows", - ] - pruneopts = "UT" - revision = "48ac38b7c8cbedd50b1613c0fccacfc7d88dfcdf" - -[[projects]] - digest = "1:436b24586f8fee329e0dd65fd67c817681420cda1d7f934345c13fe78c212a73" - name = "golang.org/x/text" - packages = [ - "collate", - "collate/build", - "encoding", - "encoding/charmap", - "encoding/htmlindex", - "encoding/internal", - "encoding/internal/identifier", - "encoding/japanese", - "encoding/korean", - "encoding/simplifiedchinese", - "encoding/traditionalchinese", - "encoding/unicode", - "internal/colltab", - "internal/gen", - "internal/tag", - "internal/triegen", - "internal/ucd", - "internal/utf8internal", - "language", - "runes", - "secure/bidirule", - "transform", - "unicode/bidi", - "unicode/cldr", - "unicode/norm", - "unicode/rangetable", - ] - pruneopts = "UT" - revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" - version = "v0.3.0" - -[[projects]] - branch = "master" - digest = "1:9fdc2b55e8e0fafe4b41884091e51e77344f7dc511c5acedcfd98200003bff90" - name = "golang.org/x/time" - packages = ["rate"] - pruneopts = "UT" - revision = "85acf8d2951cb2a3bde7632f9ff273ef0379bcbd" - -[[projects]] - branch = "master" - digest = "1:76cbf49798456f93e5a824c79eb97a96aec5bff7cd7371b810d888f7a1c33abc" - name = "golang.org/x/tools" - packages = [ - "go/ast/astutil", - "go/gcexportdata", - "go/internal/cgo", - "go/internal/gcimporter", - "go/internal/packagesdriver", - "go/packages", - "go/types/typeutil", - "imports", - "internal/fastwalk", - "internal/gopathwalk", - "internal/semver", - ] - pruneopts = "UT" - revision = "68c5ac90f574c3cf0e181d3cdde7cc60cb38fa9b" - -[[projects]] - digest = "1:6f3bd49ddf2e104e52062774d797714371fac1b8bddfd8e124ce78e6b2264a10" - name = "google.golang.org/appengine" - packages = [ - "internal", - "internal/base", - "internal/datastore", - "internal/log", - "internal/remote_api", - "internal/urlfetch", - "urlfetch", - ] - pruneopts = "UT" - revision = "e9657d882bb81064595ca3b56cbe2546bbabf7b1" - version = "v1.4.0" - -[[projects]] - digest = "1:abeb38ade3f32a92943e5be54f55ed6d6e3b6602761d74b4aab4c9dd45c18abd" - name = "gopkg.in/fsnotify.v1" - packages = ["."] - pruneopts = "UT" - revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" - source = "https://github.com/fsnotify/fsnotify.git" - version = "v1.4.7" - -[[projects]] - digest = "1:2d1fbdc6777e5408cabeb02bf336305e724b925ff4546ded0fa8715a7267922a" - name = "gopkg.in/inf.v0" - packages = ["."] - pruneopts = "UT" - revision = "d2d2541c53f18d2a059457998ce2876cc8e67cbf" - version = "v0.9.1" - -[[projects]] - branch = "v1" - digest = "1:0caa92e17bc0b65a98c63e5bc76a9e844cd5e56493f8fdbb28fad101a16254d9" - name = "gopkg.in/tomb.v1" - packages = ["."] - pruneopts = "UT" - revision = "dd632973f1e7218eb1089048e0798ec9ae7dceb8" - -[[projects]] - digest = "1:4d2e5a73dc1500038e504a8d78b986630e3626dc027bc030ba5c75da257cdb96" - name = "gopkg.in/yaml.v2" - packages = ["."] - pruneopts = "UT" - revision = "51d6538a90f86fe93ac480b35f37b2be17fef232" - version = "v2.2.2" - -[[projects]] - digest = "1:26a67eb988225c6a0600c1af0b35e795ac4d23a9c40a7aa178fa2adc0670f1f7" - name = "k8s.io/api" - packages = [ - "admission/v1beta1", - "admissionregistration/v1alpha1", - "admissionregistration/v1beta1", - "apps/v1", - "apps/v1beta1", - "apps/v1beta2", - "authentication/v1", - "authentication/v1beta1", - "authorization/v1", - "authorization/v1beta1", - "autoscaling/v1", - "autoscaling/v2beta1", - "autoscaling/v2beta2", - "batch/v1", - "batch/v1beta1", - "batch/v2alpha1", - "certificates/v1beta1", - "coordination/v1beta1", - "core/v1", - "events/v1beta1", - "extensions/v1beta1", - "networking/v1", - "policy/v1beta1", - "rbac/v1", - "rbac/v1alpha1", - "rbac/v1beta1", - "scheduling/v1alpha1", - "scheduling/v1beta1", - "settings/v1alpha1", - "storage/v1", - "storage/v1alpha1", - "storage/v1beta1", - ] - pruneopts = "UT" - revision = "b503174bad5991eb66f18247f52e41c3258f6348" - version = "kubernetes-1.12.3" - -[[projects]] - digest = "1:d9e9a6062217111c1b7c548a47bd67b2b518c8da05d124dbca74e246565c76a6" - name = "k8s.io/apiextensions-apiserver" - packages = [ - "pkg/apis/apiextensions", - "pkg/apis/apiextensions/v1beta1", - "pkg/client/clientset/clientset", - "pkg/client/clientset/clientset/scheme", - "pkg/client/clientset/clientset/typed/apiextensions/v1beta1", - ] - pruneopts = "UT" - revision = "0cd23ebeb6882bd1cdc2cb15fc7b2d72e8a86a5b" - version = "kubernetes-1.12.3" - -[[projects]] - digest = "1:14f8604409a2fe151e87f7ee6f6c954043393eb0813e3cc8bb7d309465cf7a4f" - name = "k8s.io/apimachinery" - packages = [ - "pkg/api/errors", - "pkg/api/meta", - "pkg/api/resource", - "pkg/apis/meta/internalversion", - "pkg/apis/meta/v1", - "pkg/apis/meta/v1/unstructured", - "pkg/apis/meta/v1beta1", - "pkg/conversion", - "pkg/conversion/queryparams", - "pkg/fields", - "pkg/labels", - "pkg/runtime", - "pkg/runtime/schema", - "pkg/runtime/serializer", - "pkg/runtime/serializer/json", - "pkg/runtime/serializer/protobuf", - "pkg/runtime/serializer/recognizer", - "pkg/runtime/serializer/streaming", - "pkg/runtime/serializer/versioning", - "pkg/selection", - "pkg/types", - "pkg/util/cache", - "pkg/util/clock", - "pkg/util/diff", - "pkg/util/errors", - "pkg/util/framer", - "pkg/util/httpstream", - "pkg/util/httpstream/spdy", - "pkg/util/intstr", - "pkg/util/json", - "pkg/util/mergepatch", - "pkg/util/naming", - "pkg/util/net", - "pkg/util/remotecommand", - "pkg/util/runtime", - "pkg/util/sets", - "pkg/util/strategicpatch", - "pkg/util/uuid", - "pkg/util/validation", - "pkg/util/validation/field", - "pkg/util/wait", - "pkg/util/yaml", - "pkg/version", - "pkg/watch", - "third_party/forked/golang/json", - "third_party/forked/golang/netutil", - "third_party/forked/golang/reflect", - ] - pruneopts = "UT" - revision = "eddba98df674a16931d2d4ba75edc3a389bf633a" - version = "kubernetes-1.12.3" - -[[projects]] - digest = "1:856ed15038abdf8f6df3b8636ce78cbe009fc24220b2fac3ab22791dc4653f13" - name = "k8s.io/client-go" - packages = [ - "discovery", - "dynamic", - "kubernetes", - "kubernetes/scheme", - "kubernetes/typed/admissionregistration/v1alpha1", - "kubernetes/typed/admissionregistration/v1beta1", - "kubernetes/typed/apps/v1", - "kubernetes/typed/apps/v1beta1", - "kubernetes/typed/apps/v1beta2", - "kubernetes/typed/authentication/v1", - "kubernetes/typed/authentication/v1beta1", - "kubernetes/typed/authorization/v1", - "kubernetes/typed/authorization/v1beta1", - "kubernetes/typed/autoscaling/v1", - "kubernetes/typed/autoscaling/v2beta1", - "kubernetes/typed/autoscaling/v2beta2", - "kubernetes/typed/batch/v1", - "kubernetes/typed/batch/v1beta1", - "kubernetes/typed/batch/v2alpha1", - "kubernetes/typed/certificates/v1beta1", - "kubernetes/typed/coordination/v1beta1", - "kubernetes/typed/core/v1", - "kubernetes/typed/events/v1beta1", - "kubernetes/typed/extensions/v1beta1", - "kubernetes/typed/networking/v1", - "kubernetes/typed/policy/v1beta1", - "kubernetes/typed/rbac/v1", - "kubernetes/typed/rbac/v1alpha1", - "kubernetes/typed/rbac/v1beta1", - "kubernetes/typed/scheduling/v1alpha1", - "kubernetes/typed/scheduling/v1beta1", - "kubernetes/typed/settings/v1alpha1", - "kubernetes/typed/storage/v1", - "kubernetes/typed/storage/v1alpha1", - "kubernetes/typed/storage/v1beta1", - "pkg/apis/clientauthentication", - "pkg/apis/clientauthentication/v1alpha1", - "pkg/apis/clientauthentication/v1beta1", - "pkg/version", - "plugin/pkg/client/auth/exec", - "rest", - "rest/watch", - "restmapper", - "tools/auth", - "tools/cache", - "tools/clientcmd", - "tools/clientcmd/api", - "tools/clientcmd/api/latest", - "tools/clientcmd/api/v1", - "tools/leaderelection", - "tools/leaderelection/resourcelock", - "tools/metrics", - "tools/pager", - "tools/record", - "tools/reference", - "tools/remotecommand", - "transport", - "transport/spdy", - "util/buffer", - "util/cert", - "util/connrotation", - "util/exec", - "util/flowcontrol", - "util/homedir", - "util/integer", - "util/retry", - "util/workqueue", - ] - pruneopts = "UT" - revision = "d082d5923d3cc0bfbb066ee5fbdea3d0ca79acf8" - version = "kubernetes-1.12.3" - -[[projects]] - branch = "master" - digest = "1:937e46834bfd618b223206d7e3d459bf0eb53ac7d05135bf809d73b8fafdf7f7" - name = "k8s.io/code-generator" - packages = [ - "cmd/deepcopy-gen", - "cmd/deepcopy-gen/args", - "pkg/util", - ] - pruneopts = "UT" - revision = "3a2206dd6a78497deceb3ae058417fdeb2036c7e" - -[[projects]] - branch = "master" - digest = "1:4db88b0d181fd2736b151f06eda6c776ef9ff2262d00d0eb5d6c947c03b00e91" - name = "k8s.io/gengo" - packages = [ - "args", - "examples/deepcopy-gen/generators", - "examples/set-gen/sets", - "generator", - "namer", - "parser", - "types", - ] - pruneopts = "UT" - revision = "fd15ee9cc2f77baa4f31e59e6acbf21146455073" - -[[projects]] - digest = "1:e2999bf1bb6eddc2a6aa03fe5e6629120a53088926520ca3b4765f77d7ff7eab" - name = "k8s.io/klog" - packages = ["."] - pruneopts = "UT" - revision = "a5bc97fbc634d635061f3146511332c7e313a55a" - version = "v0.1.0" - -[[projects]] - branch = "master" - digest = "1:03a96603922fc1f6895ae083e1e16d943b55ef0656b56965351bd87e7d90485f" - name = "k8s.io/kube-openapi" - packages = ["pkg/util/proto"] - pruneopts = "UT" - revision = "0317810137be915b9cf888946c6e115c1bfac693" - -[[projects]] - branch = "master" - digest = "1:e1fc4a624e8497252148125a50cec64b416d69ef69afc2244498e0c9b73060d5" - name = "k8s.io/sample-controller" - packages = ["pkg/signals"] - pruneopts = "UT" - revision = "c3a5aa93b2bfc80983f7bf509cf1fbbf1e0cfe47" - -[[projects]] - digest = "1:d5f5e5a6a100c5568ff8be9f3062f63bc1b43a993464a083700929e8d16570bb" - name = "sigs.k8s.io/controller-runtime" - packages = [ - "pkg/cache", - "pkg/cache/internal", - "pkg/client", - "pkg/client/apiutil", - "pkg/client/config", - "pkg/controller/controllerutil", - "pkg/envtest", - "pkg/envtest/printer", - "pkg/internal/recorder", - "pkg/leaderelection", - "pkg/manager", - "pkg/metrics", - "pkg/patch", - "pkg/reconcile", - "pkg/recorder", - "pkg/runtime/inject", - "pkg/runtime/log", - "pkg/runtime/scheme", - "pkg/webhook", - "pkg/webhook/admission", - "pkg/webhook/admission/builder", - "pkg/webhook/admission/types", - "pkg/webhook/internal/cert", - "pkg/webhook/internal/cert/generator", - "pkg/webhook/internal/cert/writer", - "pkg/webhook/internal/cert/writer/atomic", - "pkg/webhook/internal/metrics", - "pkg/webhook/types", - ] - pruneopts = "UT" - revision = "f6f0bc9611363b43664d08fb097ab13243ef621d" - version = "v0.1.9" - -[[projects]] - digest = "1:946a62b07e59b9abd1f475a5b5c8d590255c1d40627f386d34fd073d3de9ecb8" - name = "sigs.k8s.io/controller-tools" - packages = [ - "cmd/controller-gen", - "pkg/crd/generator", - "pkg/crd/util", - "pkg/internal/codegen", - "pkg/internal/codegen/parse", - "pkg/internal/general", - "pkg/rbac", - "pkg/util", - "pkg/webhook", - "pkg/webhook/internal", - ] - pruneopts = "UT" - revision = "950a0e88e4effb864253b3c7504b326cc83b9d11" - version = "v0.1.8" - -[[projects]] - digest = "1:9070222ca967d09b3966552a161dd4420d62315964bf5e1efd8cc4c7c30ebca8" - name = "sigs.k8s.io/testing_frameworks" - packages = [ - "integration", - "integration/addr", - "integration/internal", - ] - pruneopts = "UT" - revision = "d348cb12705b516376e0c323bacca72b00a78425" - version = "v0.1.1" - -[solve-meta] - analyzer-name = "dep" - analyzer-version = 1 - input-imports = [ - "admiralty.io/multicluster-controller/pkg/cluster", - "admiralty.io/multicluster-controller/pkg/controller", - "admiralty.io/multicluster-controller/pkg/handler", - "admiralty.io/multicluster-controller/pkg/manager", - "admiralty.io/multicluster-controller/pkg/reconcile", - "admiralty.io/multicluster-controller/pkg/reference", - "admiralty.io/multicluster-service-account/pkg/config", - "github.com/ghodss/yaml", - "github.com/go-test/deep", - "github.com/onsi/gomega", - "golang.org/x/net/context", - "gopkg.in/inf.v0", - "k8s.io/api/admissionregistration/v1beta1", - "k8s.io/api/core/v1", - "k8s.io/apimachinery/pkg/api/errors", - "k8s.io/apimachinery/pkg/api/meta", - "k8s.io/apimachinery/pkg/api/resource", - "k8s.io/apimachinery/pkg/apis/meta/v1", - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured", - "k8s.io/apimachinery/pkg/runtime", - "k8s.io/apimachinery/pkg/runtime/schema", - "k8s.io/apimachinery/pkg/types", - "k8s.io/client-go/kubernetes", - "k8s.io/client-go/kubernetes/scheme", - "k8s.io/client-go/rest", - "k8s.io/client-go/tools/remotecommand", - "k8s.io/code-generator/cmd/deepcopy-gen", - "k8s.io/sample-controller/pkg/signals", - "sigs.k8s.io/controller-runtime/pkg/client", - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil", - "sigs.k8s.io/controller-runtime/pkg/envtest", - "sigs.k8s.io/controller-runtime/pkg/manager", - "sigs.k8s.io/controller-runtime/pkg/runtime/inject", - "sigs.k8s.io/controller-runtime/pkg/runtime/scheme", - "sigs.k8s.io/controller-runtime/pkg/webhook", - "sigs.k8s.io/controller-runtime/pkg/webhook/admission", - "sigs.k8s.io/controller-runtime/pkg/webhook/admission/builder", - "sigs.k8s.io/controller-runtime/pkg/webhook/admission/types", - "sigs.k8s.io/controller-tools/cmd/controller-gen", - ] - solver-name = "gps-cdcl" - solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml deleted file mode 100644 index f13d93be..00000000 --- a/Gopkg.toml +++ /dev/null @@ -1,20 +0,0 @@ -required = [ - "k8s.io/code-generator/cmd/deepcopy-gen", - "sigs.k8s.io/controller-tools/cmd/controller-gen" -] - -[prune] - go-tests = true - unused-packages = true - -# For dependency below: Refer to issue https://github.com/golang/dep/issues/1799 -[[override]] - name = "gopkg.in/fsnotify.v1" - source = "https://github.com/fsnotify/fsnotify.git" - version="v1.4.7" - -# https://github.com/mattbaird/jsonpatch/issues/7 -# until https://github.com/kubernetes-sigs/controller-runtime/pull/289 is released -[[override]] - name = "github.com/mattbaird/jsonpatch" - source = "https://github.com/appscode/jsonpatch.git" diff --git a/PROJECT b/PROJECT deleted file mode 100644 index 94111c47..00000000 --- a/PROJECT +++ /dev/null @@ -1,3 +0,0 @@ -version: "1" -domain: admiralty.io -repo: admiralty.io/multicluster-scheduler diff --git a/README.md b/README.md index e830ef97..54956106 100644 --- a/README.md +++ b/README.md @@ -9,26 +9,9 @@ Multicluster-scheduler is a system of Kubernetes controllers that intelligently Check out [Admiralty's blog post](https://admiralty.io/blog/running-argo-workflows-across-multiple-kubernetes-clusters/) demonstrating how to run an Argo workflow across clusters to better utilize resources and combine data from different regions or clouds. -## How it Works - -![](doc/multicluster-scheduler-sequence-diagram.svg) - -Multicluster-scheduler is a system of Kubernetes controllers managed by the **scheduler**, deployed in any cluster, and its **agents**, deployed in the member clusters. The scheduler manages two controllers: the eponymous scheduler controller and the global service controller. Each agent manages six controllers: pod admission, service reroute, observations, decisions, feedback, and node pool. - -1. The **pod admission controller**, a [dynamic, mutating admission webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/), intercepts pod creation requests. If a pod is annotated with `multicluster.admiralty.io/elect=""`, its original manifest is saved as an annotation, and its containers are replaced by **proxies** that simply await success or failure signals from the feedback controller (see below). -1. The **service reroute controller** modifies services whose endpoints target proxy pods. The keys of their label selectors are prefixed with `multicluster.admiralty.io/`, to match corresponding **delegate** pods (see below). Also, the services are annotated with `io.cilium/global-service=true`, to be load-balanced across a Cilium cluster mesh. -1. The **observations controller**, a [multi-cluster controller](https://github.com/admiraltyio/multicluster-controller), watches pods (including proxy pods), services (including global services), nodes, and node pools (created by the node pool controller, see below) in the agent's cluster and reconciles corresponding **observations** in the scheduler's cluster. Observations are images of the source objects' states. -1. The **scheduler** watches proxy pod observations and reconciles delegate pod **decisions**, all in its own cluster. It decides target clusters based on the other observations. The scheduler doesn't push anything to the member clusters. -1. The **global service controller** watches global service observations (observations of services annotated with `io.cilium/global-service=true`, either by the service reroute controller or by another tool or user) and reconciles global service decisions (copies of the originals), in all clusters of the federation. -1. The **decisions controller**, another multi-cluster controller, watches pod and service decisions in the scheduler's cluster and reconciles corresponding delegates in the agent's cluster. -1. The **feedback controller** watches delegate pod observations and reconciles the corresponding proxy pods. If a delegate pod is annotated (e.g., with Argo outputs), the annotations are replicated upstream, into the corresponding proxy pod. When a delegate pod succeeds or fails, the controller kills the corresponding proxy pod's container with the proper signal. The feedback controller maintains the contract between proxy pods and their controllers, e.g., replica sets or Argo workflows. -1. The **node pool controller** automatically creates **node pool** objects in the agent's cluster. In GKE and AKS, it uses the `cloud.google.com/gke-nodepool` or `agentpool` label, respectively; in the absence of those labels, a default node pool object is created. Min/max node counts and pricing information can be updated by the user, or controlled by other tools. Custom node pool objects can also be created using label selectors. Node pool information can be used for scheduling. - -Observations, decisions, and node pools are [custom resources](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/). Node pools are defined (by CRDs) in each member cluster, whereas all observations and decisions are only defined in the scheduler's cluster. - ## Getting Started -We assume that you are a cluster admin for two clusters, associated with, e.g., the contexts "cluster1" and "cluster2" in your kubeconfig. We're going to install a basic scheduler in cluster1 and agents in cluster1 and cluster2, but the scripts can easily be adapted for other configurations. Then, we will deploy a multi-cluster NGINX. +We assume that you are a cluster admin for two clusters, associated with, e.g., the contexts "cluster1" and "cluster2" in your kubeconfig. We're going to install a basic scheduler in cluster1 and agents in cluster1 and cluster2. Then, we will deploy a multi-cluster NGINX. ```bash CLUSTER1=cluster1 # change me @@ -41,97 +24,72 @@ CLUSTER2=cluster2 # change me For cross-cluster service calls, multicluster-scheduler relies on a Cilium cluster mesh and global services. If you need this feature, [install Cilium](http://docs.cilium.io/en/stable/gettingstarted/#installation) and [set up a cluster mesh](http://docs.cilium.io/en/stable/gettingstarted/clustermesh/). If you install Cilium later, you may have to restart pods. -#### Scheduler +#### Helm -Choose a cluster to host the scheduler, download the basic scheduler's manifest and install it: +The Helm (v3) chart isn't hosted yet, so you need to clone this repository and run: ```bash -SCHEDULER_CLUSTER_NAME="$CLUSTER1" -RELEASE_URL=https://github.com/admiraltyio/multicluster-scheduler/releases/download/v0.3.0 -kubectl config use-context "$SCHEDULER_CLUSTER_NAME" -kubectl apply -f "$RELEASE_URL/scheduler.yaml" +helm install multicluster-scheduler charts/multicluster-scheduler \ + --context $CLUSTER1 \ + -f test/e2e/single-namespace/values-cluster1.yaml +helm install multicluster-scheduler charts/multicluster-scheduler \ + --context $CLUSTER2 \ + -f test/e2e/single-namespace/values-cluster2.yaml ``` -#### Federation +Note: the Helm chart is flexible enough to configure multiple federations and/or refine RBAC so clusters can't see each other's observations. While we work on properly documenting the chart, feel free reach out and/or check out the chart's own [values.yaml](charts/multicluster-scheduler/values.yaml), and some example `values.yaml` files under [test/e2e](test/e2e). -In the same cluster as the scheduler, create a namespace for the federation and, in it, a service account and role binding for each member cluster. The scheduler's cluster can be a member too. +#### Service Account Exchange -```bash -FEDERATION_NAMESPACE=foo -kubectl create namespace "$FEDERATION_NAMESPACE" -MEMBER_CLUSTER_NAMES=("$CLUSTER1" "$CLUSTER2") # Add as many member clusters as you want. -for CLUSTER_NAME in "${MEMBER_CLUSTER_NAMES[@]}"; do - kubectl create serviceaccount "$CLUSTER_NAME" \ - -n "$FEDERATION_NAMESPACE" - kubectl create rolebinding "$CLUSTER_NAME" \ - -n "$FEDERATION_NAMESPACE" \ - --serviceaccount "$FEDERATION_NAMESPACE:$CLUSTER_NAME" \ - --clusterrole multicluster-scheduler-member -done -``` - -#### Multicluster-Service-Account - -If you're already running [multicluster-service-account](https://github.com/admiraltyio/multicluster-service-account) in each member cluster, and the scheduler's cluster is known to them as `$SCHEDULER_CLUSTER_NAME`, you can skip this step and [install the agents](#agent). Otherwise, read on. - -Download the multicluster-service-account manifest and install it in each member cluster: - -```bash -MCSA_RELEASE_URL=https://github.com/admiraltyio/multicluster-service-account/releases/download/v0.3.1 -for CLUSTER_NAME in "${MEMBER_CLUSTER_NAMES[@]}"; do - kubectl --context "$CLUSTER_NAME" apply -f "$MCSA_RELEASE_URL/install.yaml" -done -``` +For agents to talk to the scheduler across cluster boundaries (via custom resource definitions, cf. [How it Works](#how-it-works)), we need to export service accounts in the scheduler's cluster as kubeconfig files and save those files inside secrets in the agents' clusters. -Then, download the kubemcsa binary and run the bootstrap command to allow member clusters to import service accounts from the scheduler's cluster: +Luckily, the `kubemcsa export` command of [multicluster-service-account](https://github.com/admiraltyio/multicluster-service-account#you-might-not-need-multicluster-service-account) can prepare the secrets for us. First, install kubemcsa (you don't need to deploy multicluster-service-account): ```bash +MCSA_RELEASE_URL=https://github.com/admiraltyio/multicluster-service-account/releases/download/v0.6.1 OS=linux # or darwin (i.e., OS X) or windows ARCH=amd64 # if you're on a different platform, you must know how to build from source curl -Lo kubemcsa "$MCSA_RELEASE_URL/kubemcsa-$OS-$ARCH" chmod +x kubemcsa -sudo mv kubemcsa /usr/local/bin - -for CLUSTER_NAME in "${MEMBER_CLUSTER_NAMES[@]}"; do - kubemcsa bootstrap "$CLUSTER_NAME" "$SCHEDULER_CLUSTER_NAME" -done ``` -#### Agent - -Download the agent manifest and install it in each member cluster: +Then, run `kubemcsa export` to generate templates for secrets containing kubeconfigs equivalent to the `c1` and `c2` service accounts created by Helm in cluster1, and apply the templates with kubectl in cluster1 and cluster2, respectively: ```bash -curl -LO $RELEASE_URL/agent.yaml -for CLUSTER_NAME in "${MEMBER_CLUSTER_NAMES[@]}"; do - sed -e "s/SCHEDULER_CLUSTER_NAME/$SCHEDULER_CLUSTER_NAME/g" \ - -e "s/FEDERATION_NAMESPACE/$FEDERATION_NAMESPACE/g" \ - -e "s/CLUSTER_NAME/$CLUSTER_NAME/g" \ - agent.yaml > "agent-$CLUSTER_NAME.yaml" - kubectl --context "$CLUSTER_NAME" apply -f "agent-$CLUSTER_NAME.yaml" -done +./kubemcsa export --context $CLUSTER1 c1 --as remote | kubectl --context $CLUSTER1 apply -f - +./kubemcsa export --context $CLUSTER1 c2 --as remote | kubectl --context $CLUSTER2 apply -f - ``` -Check that node pool objects have been created in the agents' clusters and observations appear in the scheduler's cluster: +Note: you may wonder why the agent in cluster1 needs a kubeconfig as it runs in the same cluster as the scheduler. We simply like symmetry and didn't want to make the agent's configuration special in that case. + +#### Verification + +After a minute, check that node pool objects have been created in the agents' clusters and observations appear in the scheduler's cluster: ```bash -for CLUSTER_NAME in "${MEMBER_CLUSTER_NAMES[@]}"; do - kubectl --context "$CLUSTER_NAME" get nodepools # or np -done -kubectl -n "$FEDERATION_NAMESPACE" get nodepoolobservations # or npobs -kubectl -n "$FEDERATION_NAMESPACE" get nodeobservations # or nodeobs -kubectl -n "$FEDERATION_NAMESPACE" get podobservations # or podobs -kubectl -n "$FEDERATION_NAMESPACE" get serviceobservations # or svcobs +kubectl --context $CLUSTER1 get nodepools # or np +kubectl --context $CLUSTER2 get nodepools # or np + +kubectl config use-context $CLUSTER1 +kubectl get nodepoolobservations # or npobs +kubectl get nodeobservations # or nodeobs +kubectl get podobservations # or podobs +kubectl get serviceobservations # or svcobs # or, by category -kubectl -n "$FEDERATION_NAMESPACE" get observations --show-kind # or obs +kubectl get observations --show-kind # or obs ``` ### Example -Multicluster-scheduler's pod admission controller operates in namespaces labeled with `multicluster-scheduler=enabled`. In any of the member cluster, e.g., cluster2, label the `default` namespace. Then, deploy NGINX in it with the election annotation on the pod template: +Multicluster-scheduler's pod admission controller operates in namespaces labeled with `multicluster-scheduler=enabled`. In any of the member cluster, e.g., cluster2, label the `default` namespace: ```bash kubectl --context "$CLUSTER2" label namespace default multicluster-scheduler=enabled +``` + +Then, deploy NGINX in it with the election annotation on the pod template: + +```bash cat < ../multicluster-controller + // admiralty.io/multicluster-service-account => ../multicluster-service-account + github.com/appscode/jsonpatch => gomodules.xyz/jsonpatch/v2 v2.0.0 + k8s.io/api => k8s.io/api v0.0.0-20190222213804-5cb15d344471 // release-1.13 + k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20190221213512-86fb29eff628 // release-1.13 +) + +require ( + admiralty.io/multicluster-controller v0.3.1 + admiralty.io/multicluster-service-account v0.6.1 + github.com/appscode/jsonpatch v0.0.0-00010101000000-000000000000 // indirect + github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c // indirect + github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484 // indirect + github.com/ghodss/yaml v1.0.0 + github.com/go-logr/zapr v0.1.1 // indirect + github.com/go-test/deep v1.0.1 + github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect + github.com/googleapis/gnostic v0.3.0 // indirect + github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect + github.com/imdario/mergo v0.3.7 // indirect + github.com/onsi/gomega v1.4.3 + go.uber.org/atomic v1.4.0 // indirect + go.uber.org/multierr v1.2.0 // indirect + golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 + golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect + gomodules.xyz/jsonpatch/v2 v2.0.1 // indirect + gopkg.in/inf.v0 v0.9.1 + k8s.io/api v0.0.0-20191025225708-5524a3672fbb + k8s.io/apiextensions-apiserver v0.0.0-20191026071228-81c2f4fbaa0d // indirect + k8s.io/apimachinery v0.0.0-20191025225532-af6325b3a843 + k8s.io/client-go v10.0.0+incompatible + k8s.io/sample-controller v0.0.0-20190625130054-294bc0f66822 + sigs.k8s.io/controller-runtime v0.1.12 + sigs.k8s.io/testing_frameworks v0.1.2 // indirect + sigs.k8s.io/yaml v1.1.0 +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..c3522a22 --- /dev/null +++ b/go.sum @@ -0,0 +1,822 @@ +admiralty.io/multicluster-controller v0.2.0 h1:ajPasPVjeT9gKI9egEUhZOWXIkJZAZJw4r1f1gSovhI= +admiralty.io/multicluster-controller v0.2.0/go.mod h1:hpWrIHZz6R7rHbm034MouEbhMIs4B4L1AlBeaK9dvfE= +admiralty.io/multicluster-controller v0.3.1 h1:cPSpNCIc8quRgzRgFIOBtSDgUyvCuHVr72+v20goQ8k= +admiralty.io/multicluster-controller v0.3.1/go.mod h1:AErm5eiAtxaHow+owSv3FWjSwCGi75U6/DFVoGOeepQ= +admiralty.io/multicluster-service-account v0.1.0/go.mod h1:fkgbwbknpY4gCmZwAkmy7I+H9GNqE30/HIldfoOOAH0= +admiralty.io/multicluster-service-account v0.4.1 h1:ygJ6XGpKsipuKvoRgsc2Ft06stHz8vU1ByIy5TM5E7s= +admiralty.io/multicluster-service-account v0.4.1/go.mod h1:fkgbwbknpY4gCmZwAkmy7I+H9GNqE30/HIldfoOOAH0= +admiralty.io/multicluster-service-account v0.6.1 h1:uIdAjUYHKBT3nmAh+EXOTMxtUiV5KuviIN1CGUKRlX0= +admiralty.io/multicluster-service-account v0.6.1/go.mod h1:V6XeGFNGhtgzFVnI+49Ur5pLWzwJQmAKLt+IiK4gvxI= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.28.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0 h1:ROfEUZz+Gh5pa62DJWXSaonyu3StP6EA6lPEXPI6mCo= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-autorest v10.15.5+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest v11.1.2+incompatible h1:viZ3tV5l4gE2Sw0xrasFHytCGtzYCrT+um/rrSQ1BfA= +github.com/Azure/go-autorest v11.1.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= +github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= +github.com/Azure/go-autorest/autorest/date v0.1.0 h1:YGrhWfrgtFs84+h0o46rJrlmsZtyZRg470CqAXTZaGM= +github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1GnWeHDdaNKY= +github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= +github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46 h1:lsxEuwrXEAokXB9qhlbKWPpo3KMLZQ5WB5WLQRW1uq0= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.0 h1:rmGxhojJlM0tuKtfdvliR84CFHljx9ag64t2xmVkjK4= +github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c/go.mod h1:XGLbWH/ujMcbPbhZq52Nv6UrCghb1yGn//133kEsvDk= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/codegangsta/negroni v1.0.0/go.mod h1:v0y3T5G7Y1UlFfyxFn/QLRU4a2EuNau2iZY63YTKWo0= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7 h1:u9SHYsPQNyt5tgDm3YN7+9dYrpK96E5wFilTFWIDZOM= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0 h1:w3NnFcKR5241cfmQU5ZZAsf0xcpId6mWOupTvJlUX2U= +github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c h1:ZfSZ3P3BedhKGUhzj7BQlPSU4OvT6tfOKe3DVHzOA7s= +github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v0.0.0-20180713052910-9f541cc9db5d/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484 h1:pEtiCjIXx3RvGjlUJuCNxNOw0MNblyR9Wi+vJGBFh+8= +github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= +github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy03t/bUadywsbyrQwCqZeNIEX6M1OtSZOM= +github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633 h1:H2pdYOb3KQ1/YsqVWoWNLQO+fusocsw354rqGTZtAgw= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= +github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= +github.com/go-logr/zapr v0.1.1 h1:qXBXPDdNncunGs7XeEpsJt8wCjYBygluzfdLO0G5baE= +github.com/go-logr/zapr v0.1.1/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= +github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= +github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= +github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= +github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= +github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= +github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= +github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= +github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= +github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= +github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= +github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= +github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= +github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= +github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= +github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= +github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= +github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= +github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= +github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= +github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= +github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= +github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= +github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= +github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-test/deep v1.0.1 h1:UQhStjbkDClarlmv0am7OXXO4/GaPdCGiUiMTvi28sg= +github.com/go-test/deep v1.0.1/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= +github.com/gobuffalo/buffalo v0.12.8-0.20181004233540-fac9bb505aa8/go.mod h1:sLyT7/dceRXJUxSsE813JTQtA3Eb1vjxWfo/N//vXIY= +github.com/gobuffalo/buffalo v0.13.0/go.mod h1:Mjn1Ba9wpIbpbrD+lIDMy99pQ0H0LiddMIIDGse7qT4= +github.com/gobuffalo/buffalo-plugins v1.0.2/go.mod h1:pOp/uF7X3IShFHyobahTkTLZaeUXwb0GrUTb9ngJWTs= +github.com/gobuffalo/buffalo-plugins v1.0.4/go.mod h1:pWS1vjtQ6uD17MVFWf7i3zfThrEKWlI5+PYLw/NaDB4= +github.com/gobuffalo/buffalo-plugins v1.4.3/go.mod h1:uCzTY0woez4nDMdQjkcOYKanngeUVRO2HZi7ezmAjWY= +github.com/gobuffalo/buffalo-plugins v1.5.1/go.mod h1:jbmwSZK5+PiAP9cC09VQOrGMZFCa/P0UMlIS3O12r5w= +github.com/gobuffalo/buffalo-plugins v1.6.4/go.mod h1:/+N1aophkA2jZ1ifB2O3Y9yGwu6gKOVMtUmJnbg+OZI= +github.com/gobuffalo/buffalo-plugins v1.6.5/go.mod h1:0HVkbgrVs/MnPZ/FOseDMVanCTm2RNcdM0PuXcL1NNI= +github.com/gobuffalo/buffalo-plugins v1.6.7/go.mod h1:ZGZRkzz2PiKWHs0z7QsPBOTo2EpcGRArMEym6ghKYgk= +github.com/gobuffalo/buffalo-plugins v1.6.9/go.mod h1:yYlYTrPdMCz+6/+UaXg5Jm4gN3xhsvsQ2ygVatZV5vw= +github.com/gobuffalo/buffalo-plugins v1.6.11/go.mod h1:eAA6xJIL8OuynJZ8amXjRmHND6YiusVAaJdHDN1Lu8Q= +github.com/gobuffalo/buffalo-plugins v1.8.2/go.mod h1:9te6/VjEQ7pKp7lXlDIMqzxgGpjlKoAcAANdCgoR960= +github.com/gobuffalo/buffalo-pop v1.0.5/go.mod h1:Fw/LfFDnSmB/vvQXPvcXEjzP98Tc+AudyNWUBWKCwQ8= +github.com/gobuffalo/envy v1.6.4/go.mod h1:Abh+Jfw475/NWtYMEt+hnJWRiC8INKWibIMyNt1w2Mc= +github.com/gobuffalo/envy v1.6.5/go.mod h1:N+GkhhZ/93bGZc6ZKhJLP6+m+tCNPKwgSpH9kaifseQ= +github.com/gobuffalo/envy v1.6.6/go.mod h1:N+GkhhZ/93bGZc6ZKhJLP6+m+tCNPKwgSpH9kaifseQ= +github.com/gobuffalo/envy v1.6.7/go.mod h1:N+GkhhZ/93bGZc6ZKhJLP6+m+tCNPKwgSpH9kaifseQ= +github.com/gobuffalo/envy v1.6.8/go.mod h1:N+GkhhZ/93bGZc6ZKhJLP6+m+tCNPKwgSpH9kaifseQ= +github.com/gobuffalo/envy v1.6.9/go.mod h1:N+GkhhZ/93bGZc6ZKhJLP6+m+tCNPKwgSpH9kaifseQ= +github.com/gobuffalo/envy v1.6.10/go.mod h1:X0CFllQjTV5ogsnUrg+Oks2yTI+PU2dGYBJOEI2D1Uo= +github.com/gobuffalo/envy v1.6.11/go.mod h1:Fiq52W7nrHGDggFPhn2ZCcHw4u/rqXkqo+i7FB6EAcg= +github.com/gobuffalo/events v1.0.3/go.mod h1:Txo8WmqScapa7zimEQIwgiJBvMECMe9gJjsKNPN3uZw= +github.com/gobuffalo/events v1.0.7/go.mod h1:z8txf6H9jWhQ5Scr7YPLWg/cgXBRj8Q4uYI+rsVCCSQ= +github.com/gobuffalo/events v1.0.8/go.mod h1:A5KyqT1sA+3GJiBE4QKZibse9mtOcI9nw8gGrDdqYGs= +github.com/gobuffalo/events v1.1.3/go.mod h1:9yPGWYv11GENtzrIRApwQRMYSbUgCsZ1w6R503fCfrk= +github.com/gobuffalo/events v1.1.4/go.mod h1:09/YRRgZHEOts5Isov+g9X2xajxdvOAcUuAHIX/O//A= +github.com/gobuffalo/events v1.1.5/go.mod h1:3YUSzgHfYctSjEjLCWbkXP6djH2M+MLaVRzb4ymbAK0= +github.com/gobuffalo/events v1.1.7/go.mod h1:6fGqxH2ing5XMb3EYRq9LEkVlyPGs4oO/eLzh+S8CxY= +github.com/gobuffalo/events v1.1.8/go.mod h1:UFy+W6X6VbCWS8k2iT81HYX65dMtiuVycMy04cplt/8= +github.com/gobuffalo/fizz v1.0.12/go.mod h1:C0sltPxpYK8Ftvf64kbsQa2yiCZY4RZviurNxXdAKwc= +github.com/gobuffalo/flect v0.0.0-20180907193754-dc14d8acaf9f/go.mod h1:rCiQgmAE4axgBNl3jZWzS5rETRYTGOsrixTRaCPzNdA= +github.com/gobuffalo/flect v0.0.0-20181002182613-4571df4b1daf/go.mod h1:rCiQgmAE4axgBNl3jZWzS5rETRYTGOsrixTRaCPzNdA= +github.com/gobuffalo/flect v0.0.0-20181007231023-ae7ed6bfe683/go.mod h1:rCiQgmAE4axgBNl3jZWzS5rETRYTGOsrixTRaCPzNdA= +github.com/gobuffalo/flect v0.0.0-20181018182602-fd24a256709f/go.mod h1:rCiQgmAE4axgBNl3jZWzS5rETRYTGOsrixTRaCPzNdA= +github.com/gobuffalo/flect v0.0.0-20181019110701-3d6f0b585514/go.mod h1:rCiQgmAE4axgBNl3jZWzS5rETRYTGOsrixTRaCPzNdA= +github.com/gobuffalo/flect v0.0.0-20181024204909-8f6be1a8c6c2/go.mod h1:rCiQgmAE4axgBNl3jZWzS5rETRYTGOsrixTRaCPzNdA= +github.com/gobuffalo/flect v0.0.0-20181104133451-1f6e9779237a/go.mod h1:rCiQgmAE4axgBNl3jZWzS5rETRYTGOsrixTRaCPzNdA= +github.com/gobuffalo/flect v0.0.0-20181114183036-47375f6d8328/go.mod h1:0HvNbHdfh+WOvDSIASqJOSxTOWSxCCUF++k/Y53v9rI= +github.com/gobuffalo/genny v0.0.0-20180924032338-7af3a40f2252/go.mod h1:tUTQOogrr7tAQnhajMSH6rv1BVev34H2sa1xNHMy94g= +github.com/gobuffalo/genny v0.0.0-20181003150629-3786a0744c5d/go.mod h1:WAd8HmjMVrnkAZbmfgH5dLBUchsZfqzp/WS5sQz+uTM= +github.com/gobuffalo/genny v0.0.0-20181005145118-318a41a134cc/go.mod h1:WAd8HmjMVrnkAZbmfgH5dLBUchsZfqzp/WS5sQz+uTM= +github.com/gobuffalo/genny v0.0.0-20181007153042-b8de7d566757/go.mod h1:+oG5Ljrw04czAHbPXREwaFojJbpUvcIy4DiOnbEJFTA= +github.com/gobuffalo/genny v0.0.0-20181012161047-33e5f43d83a6/go.mod h1:+oG5Ljrw04czAHbPXREwaFojJbpUvcIy4DiOnbEJFTA= +github.com/gobuffalo/genny v0.0.0-20181017160347-90a774534246/go.mod h1:+oG5Ljrw04czAHbPXREwaFojJbpUvcIy4DiOnbEJFTA= +github.com/gobuffalo/genny v0.0.0-20181024195656-51392254bf53/go.mod h1:o9GEH5gn5sCKLVB5rHFC4tq40rQ3VRUzmx6WwmaqISE= +github.com/gobuffalo/genny v0.0.0-20181025145300-af3f81d526b8/go.mod h1:uZ1fFYvdcP8mu0B/Ynarf6dsGvp7QFIpk/QACUuFUVI= +github.com/gobuffalo/genny v0.0.0-20181027191429-94d6cfb5c7fc/go.mod h1:x7SkrQQBx204Y+O9EwRXeszLJDTaWN0GnEasxgLrQTA= +github.com/gobuffalo/genny v0.0.0-20181027195209-3887b7171c4f/go.mod h1:JbKx8HSWICu5zyqWOa0dVV1pbbXOHusrSzQUprW6g+w= +github.com/gobuffalo/genny v0.0.0-20181106193839-7dcb0924caf1/go.mod h1:x61yHxvbDCgQ/7cOAbJCacZQuHgB0KMSzoYcw5debjU= +github.com/gobuffalo/genny v0.0.0-20181107223128-f18346459dbe/go.mod h1:utQD3aKKEsdb03oR+Vi/6ztQb1j7pO10N3OBoowRcSU= +github.com/gobuffalo/genny v0.0.0-20181114215459-0a4decd77f5d/go.mod h1:kN2KZ8VgXF9VIIOj/GM0Eo7YK+un4Q3tTreKOf0q1ng= +github.com/gobuffalo/genny v0.0.0-20181119162812-e8ff4adce8bb/go.mod h1:BA9htSe4bZwBDJLe8CUkoqkypq3hn3+CkoHqVOW718E= +github.com/gobuffalo/genny v0.0.0-20181127225641-2d959acc795b/go.mod h1:l54xLXNkteX/PdZ+HlgPk1qtcrgeOr3XUBBPDbH+7CQ= +github.com/gobuffalo/genny v0.0.0-20181128191930-77e34f71ba2a/go.mod h1:FW/D9p7cEEOqxYA71/hnrkOWm62JZ5ZNxcNIVJEaWBU= +github.com/gobuffalo/genny v0.0.0-20181203165245-fda8bcce96b1/go.mod h1:wpNSANu9UErftfiaAlz1pDZclrYzLtO5lALifODyjuM= +github.com/gobuffalo/genny v0.0.0-20181203201232-849d2c9534ea/go.mod h1:wpNSANu9UErftfiaAlz1pDZclrYzLtO5lALifODyjuM= +github.com/gobuffalo/genny v0.0.0-20181206121324-d6fb8a0dbe36/go.mod h1:wpNSANu9UErftfiaAlz1pDZclrYzLtO5lALifODyjuM= +github.com/gobuffalo/genny v0.0.0-20181207164119-84844398a37d/go.mod h1:y0ysCHGGQf2T3vOhCrGHheYN54Y/REj0ayd0Suf4C/8= +github.com/gobuffalo/github_flavored_markdown v1.0.4/go.mod h1:uRowCdK+q8d/RF0Kt3/DSalaIXbb0De/dmTqMQdkQ4I= +github.com/gobuffalo/github_flavored_markdown v1.0.5/go.mod h1:U0643QShPF+OF2tJvYNiYDLDGDuQmJZXsf/bHOJPsMY= +github.com/gobuffalo/github_flavored_markdown v1.0.7/go.mod h1:w93Pd9Lz6LvyQXEG6DktTPHkOtCbr+arAD5mkwMzXLI= +github.com/gobuffalo/httptest v1.0.2/go.mod h1:7T1IbSrg60ankme0aDLVnEY0h056g9M1/ZvpVThtB7E= +github.com/gobuffalo/licenser v0.0.0-20180924033006-eae28e638a42/go.mod h1:Ubo90Np8gpsSZqNScZZkVXXAo5DGhTb+WYFIjlnog8w= +github.com/gobuffalo/licenser v0.0.0-20181025145548-437d89de4f75/go.mod h1:x3lEpYxkRG/XtGCUNkio+6RZ/dlOvLzTI9M1auIwFcw= +github.com/gobuffalo/licenser v0.0.0-20181027200154-58051a75da95/go.mod h1:BzhaaxGd1tq1+OLKObzgdCV9kqVhbTulxOpYbvMQWS0= +github.com/gobuffalo/licenser v0.0.0-20181109171355-91a2a7aac9a7/go.mod h1:m+Ygox92pi9bdg+gVaycvqE8RVSjZp7mWw75+K5NPHk= +github.com/gobuffalo/licenser v0.0.0-20181128165715-cc7305f8abed/go.mod h1:oU9F9UCE+AzI/MueCKZamsezGOOHfSirltllOVeRTAE= +github.com/gobuffalo/licenser v0.0.0-20181203160806-fe900bbede07/go.mod h1:ph6VDNvOzt1CdfaWC+9XwcBnlSTBz2j49PBwum6RFaU= +github.com/gobuffalo/logger v0.0.0-20181022175615-46cfb361fc27/go.mod h1:8sQkgyhWipz1mIctHF4jTxmJh1Vxhp7mP8IqbljgJZo= +github.com/gobuffalo/logger v0.0.0-20181027144941-73d08d2bb969/go.mod h1:7uGg2duHKpWnN4+YmyKBdLXfhopkAdVM6H3nKbyFbz8= +github.com/gobuffalo/logger v0.0.0-20181027193913-9cf4dd0efe46/go.mod h1:7uGg2duHKpWnN4+YmyKBdLXfhopkAdVM6H3nKbyFbz8= +github.com/gobuffalo/logger v0.0.0-20181109185836-3feeab578c17/go.mod h1:oNErH0xLe+utO+OW8ptXMSA5DkiSEDW1u3zGIt8F9Ew= +github.com/gobuffalo/logger v0.0.0-20181117211126-8e9b89b7c264/go.mod h1:5etB91IE0uBlw9k756fVKZJdS+7M7ejVhmpXXiSFj0I= +github.com/gobuffalo/logger v0.0.0-20181127160119-5b956e21995c/go.mod h1:+HxKANrR9VGw9yN3aOAppJKvhO05ctDi63w4mDnKv2U= +github.com/gobuffalo/makr v1.1.5/go.mod h1:Y+o0btAH1kYAMDJW/TX3+oAXEu0bmSLLoC9mIFxtzOw= +github.com/gobuffalo/mapi v1.0.0/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/meta v0.0.0-20181018155829-df62557efcd3/go.mod h1:XTTOhwMNryif3x9LkTTBO/Llrveezd71u3quLd0u7CM= +github.com/gobuffalo/meta v0.0.0-20181018192820-8c6cef77dab3/go.mod h1:E94EPzx9NERGCY69UWlcj6Hipf2uK/vnfrF4QD0plVE= +github.com/gobuffalo/meta v0.0.0-20181025145500-3a985a084b0a/go.mod h1:YDAKBud2FP7NZdruCSlmTmDOZbVSa6bpK7LJ/A/nlKg= +github.com/gobuffalo/meta v0.0.0-20181114191255-b130ebedd2f7/go.mod h1:K6cRZ29ozr4Btvsqkjvg5nDFTLOgTqf03KA70Ks0ypE= +github.com/gobuffalo/meta v0.0.0-20181127070345-0d7e59dd540b/go.mod h1:RLO7tMvE0IAKAM8wny1aN12pvEKn7EtkBLkUZR00Qf8= +github.com/gobuffalo/mw-basicauth v1.0.3/go.mod h1:dg7+ilMZOKnQFHDefUzUHufNyTswVUviCBgF244C1+0= +github.com/gobuffalo/mw-contenttype v0.0.0-20180802152300-74f5a47f4d56/go.mod h1:7EvcmzBbeCvFtQm5GqF9ys6QnCxz2UM1x0moiWLq1No= +github.com/gobuffalo/mw-csrf v0.0.0-20180802151833-446ff26e108b/go.mod h1:sbGtb8DmDZuDUQoxjr8hG1ZbLtZboD9xsn6p77ppcHo= +github.com/gobuffalo/mw-forcessl v0.0.0-20180802152810-73921ae7a130/go.mod h1:JvNHRj7bYNAMUr/5XMkZaDcw3jZhUZpsmzhd//FFWmQ= +github.com/gobuffalo/mw-i18n v0.0.0-20180802152014-e3060b7e13d6/go.mod h1:91AQfukc52A6hdfIfkxzyr+kpVYDodgAeT5cjX1UIj4= +github.com/gobuffalo/mw-paramlogger v0.0.0-20181005191442-d6ee392ec72e/go.mod h1:6OJr6VwSzgJMqWMj7TYmRUqzNe2LXu/W1rRW4MAz/ME= +github.com/gobuffalo/mw-tokenauth v0.0.0-20181001105134-8545f626c189/go.mod h1:UqBF00IfKvd39ni5+yI5MLMjAf4gX7cDKN/26zDOD6c= +github.com/gobuffalo/packd v0.0.0-20181027182251-01ad393492c8/go.mod h1:SmdBdhj6uhOsg1Ui4SFAyrhuc7U4VCildosO5IDJ3lc= +github.com/gobuffalo/packd v0.0.0-20181027190505-aafc0d02c411/go.mod h1:SmdBdhj6uhOsg1Ui4SFAyrhuc7U4VCildosO5IDJ3lc= +github.com/gobuffalo/packd v0.0.0-20181027194105-7ae579e6d213/go.mod h1:SmdBdhj6uhOsg1Ui4SFAyrhuc7U4VCildosO5IDJ3lc= +github.com/gobuffalo/packd v0.0.0-20181031195726-c82734870264/go.mod h1:Yf2toFaISlyQrr5TfO3h6DB9pl9mZRmyvBGQb/aQ/pI= +github.com/gobuffalo/packd v0.0.0-20181104210303-d376b15f8e96/go.mod h1:Yf2toFaISlyQrr5TfO3h6DB9pl9mZRmyvBGQb/aQ/pI= +github.com/gobuffalo/packd v0.0.0-20181111195323-b2e760a5f0ff/go.mod h1:Yf2toFaISlyQrr5TfO3h6DB9pl9mZRmyvBGQb/aQ/pI= +github.com/gobuffalo/packd v0.0.0-20181114190715-f25c5d2471d7/go.mod h1:Yf2toFaISlyQrr5TfO3h6DB9pl9mZRmyvBGQb/aQ/pI= +github.com/gobuffalo/packd v0.0.0-20181124090624-311c6248e5fb/go.mod h1:Foenia9ZvITEvG05ab6XpiD5EfBHPL8A6hush8SJ0o8= +github.com/gobuffalo/packd v0.0.0-20181207120301-c49825f8f6f4/go.mod h1:LYc0TGKFBBFTRC9dg2pcRcMqGCTMD7T2BIMP7OBuQAA= +github.com/gobuffalo/packr v1.13.7/go.mod h1:KkinLIn/n6+3tVXMwg6KkNvWwVsrRAz4ph+jgpk3Z24= +github.com/gobuffalo/packr v1.15.0/go.mod h1:t5gXzEhIviQwVlNx/+3SfS07GS+cZ2hn76WLzPp6MGI= +github.com/gobuffalo/packr v1.15.1/go.mod h1:IeqicJ7jm8182yrVmNbM6PR4g79SjN9tZLH8KduZZwE= +github.com/gobuffalo/packr v1.19.0/go.mod h1:MstrNkfCQhd5o+Ct4IJ0skWlxN8emOq8DsoT1G98VIU= +github.com/gobuffalo/packr v1.20.0/go.mod h1:JDytk1t2gP+my1ig7iI4NcVaXr886+N0ecUga6884zw= +github.com/gobuffalo/packr v1.21.0/go.mod h1:H00jGfj1qFKxscFJSw8wcL4hpQtPe1PfU2wa6sg/SR0= +github.com/gobuffalo/packr/v2 v2.0.0-rc.8/go.mod h1:y60QCdzwuMwO2R49fdQhsjCPv7tLQFR0ayzxxla9zes= +github.com/gobuffalo/packr/v2 v2.0.0-rc.10/go.mod h1:4CWWn4I5T3v4c1OsJ55HbHlUEKNWMITG5iIkdr4Px4w= +github.com/gobuffalo/packr/v2 v2.0.0-rc.11/go.mod h1:JoieH/3h3U4UmatmV93QmqyPUdf4wVM9HELaHEu+3fk= +github.com/gobuffalo/plush v3.7.16+incompatible/go.mod h1:rQ4zdtUUyZNqULlc6bqd5scsPfLKfT0+TGMChgduDvI= +github.com/gobuffalo/plush v3.7.20+incompatible/go.mod h1:rQ4zdtUUyZNqULlc6bqd5scsPfLKfT0+TGMChgduDvI= +github.com/gobuffalo/plush v3.7.21+incompatible/go.mod h1:rQ4zdtUUyZNqULlc6bqd5scsPfLKfT0+TGMChgduDvI= +github.com/gobuffalo/plush v3.7.22+incompatible/go.mod h1:rQ4zdtUUyZNqULlc6bqd5scsPfLKfT0+TGMChgduDvI= +github.com/gobuffalo/plush v3.7.23+incompatible/go.mod h1:rQ4zdtUUyZNqULlc6bqd5scsPfLKfT0+TGMChgduDvI= +github.com/gobuffalo/plush v3.7.30+incompatible/go.mod h1:rQ4zdtUUyZNqULlc6bqd5scsPfLKfT0+TGMChgduDvI= +github.com/gobuffalo/plush v3.7.32+incompatible/go.mod h1:rQ4zdtUUyZNqULlc6bqd5scsPfLKfT0+TGMChgduDvI= +github.com/gobuffalo/plushgen v0.0.0-20181128164830-d29dcb966cb2/go.mod h1:r9QwptTFnuvSaSRjpSp4S2/4e2D3tJhARYbvEBcKSb4= +github.com/gobuffalo/plushgen v0.0.0-20181203163832-9fc4964505c2/go.mod h1:opEdT33AA2HdrIwK1aibqnTJDVVKXC02Bar/GT1YRVs= +github.com/gobuffalo/plushgen v0.0.0-20181207152837-eedb135bd51b/go.mod h1:Lcw7HQbEVm09sAQrCLzIxuhFbB3nAgp4c55E+UlynR0= +github.com/gobuffalo/pop v4.8.2+incompatible/go.mod h1:DwBz3SD5SsHpTZiTubcsFWcVDpJWGsxjVjMPnkiThWg= +github.com/gobuffalo/pop v4.8.3+incompatible/go.mod h1:DwBz3SD5SsHpTZiTubcsFWcVDpJWGsxjVjMPnkiThWg= +github.com/gobuffalo/pop v4.8.4+incompatible/go.mod h1:DwBz3SD5SsHpTZiTubcsFWcVDpJWGsxjVjMPnkiThWg= +github.com/gobuffalo/release v1.0.35/go.mod h1:VtHFAKs61vO3wboCec5xr9JPTjYyWYcvaM3lclkc4x4= +github.com/gobuffalo/release v1.0.38/go.mod h1:VtHFAKs61vO3wboCec5xr9JPTjYyWYcvaM3lclkc4x4= +github.com/gobuffalo/release v1.0.42/go.mod h1:RPs7EtafH4oylgetOJpGP0yCZZUiO4vqHfTHJjSdpug= +github.com/gobuffalo/release v1.0.52/go.mod h1:RPs7EtafH4oylgetOJpGP0yCZZUiO4vqHfTHJjSdpug= +github.com/gobuffalo/release v1.0.53/go.mod h1:FdF257nd8rqhNaqtDWFGhxdJ/Ig4J7VcS3KL7n/a+aA= +github.com/gobuffalo/release v1.0.54/go.mod h1:Pe5/RxRa/BE8whDpGfRqSI7D1a0evGK1T4JDm339tJc= +github.com/gobuffalo/release v1.0.61/go.mod h1:mfIO38ujUNVDlBziIYqXquYfBF+8FDHUjKZgYC1Hj24= +github.com/gobuffalo/release v1.0.72/go.mod h1:NP5NXgg/IX3M5XmHmWR99D687/3Dt9qZtTK/Lbwc1hU= +github.com/gobuffalo/release v1.1.1/go.mod h1:Sluak1Xd6kcp6snkluR1jeXAogdJZpFFRzTYRs/2uwg= +github.com/gobuffalo/shoulders v1.0.1/go.mod h1:V33CcVmaQ4gRUmHKwq1fiTXuf8Gp/qjQBUL5tHPmvbA= +github.com/gobuffalo/syncx v0.0.0-20181120191700-98333ab04150/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= +github.com/gobuffalo/syncx v0.0.0-20181120194010-558ac7de985f/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= +github.com/gobuffalo/tags v2.0.11+incompatible/go.mod h1:9XmhOkyaB7UzvuY4UoZO4s67q8/xRMVJEaakauVQYeY= +github.com/gobuffalo/tags v2.0.14+incompatible/go.mod h1:9XmhOkyaB7UzvuY4UoZO4s67q8/xRMVJEaakauVQYeY= +github.com/gobuffalo/uuid v2.0.3+incompatible/go.mod h1:ErhIzkRhm0FtRuiE/PeORqcw4cVi1RtSpnwYrxuvkfE= +github.com/gobuffalo/uuid v2.0.4+incompatible/go.mod h1:ErhIzkRhm0FtRuiE/PeORqcw4cVi1RtSpnwYrxuvkfE= +github.com/gobuffalo/uuid v2.0.5+incompatible/go.mod h1:ErhIzkRhm0FtRuiE/PeORqcw4cVi1RtSpnwYrxuvkfE= +github.com/gobuffalo/validate v2.0.3+incompatible/go.mod h1:N+EtDe0J8252BgfzQUChBgfd6L93m9weay53EWFVsMM= +github.com/gobuffalo/x v0.0.0-20181003152136-452098b06085/go.mod h1:WevpGD+5YOreDJznWevcn8NTmQEW5STSBgIkpkjzqXc= +github.com/gobuffalo/x v0.0.0-20181007152206-913e47c59ca7/go.mod h1:9rDPXaB3kXdKWzMc4odGQQdG2e2DIEmANy5aSJ9yesY= +github.com/gofrs/uuid v3.1.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415 h1:WSBJMqJbLxsn+bTCPyPYZfqHdJmc8MK4wrBjMft6BAM= +github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I= +github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20181024230925-c65c006176ff/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/btree v0.0.0-20160524151835-7d79101e329e h1:JHB7F/4TJCrYBW8+GZO8VkWDj1jxcWuCl6uxKODiyi4= +github.com/google/btree v0.0.0-20160524151835-7d79101e329e/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= +github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= +github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k= +github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.3.0 h1:CcQijm0XKekKjP/YCz28LXVSpgguuB+nCxaSjCe09y0= +github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/gophercloud/gophercloud v0.0.0-20190126172459-c818fa66e4c8/go.mod h1:3WdhXV3rUYy9p6AUW8d94kr+HS62Y4VL9mBnFxsD8q4= +github.com/gophercloud/gophercloud v0.0.0-20190406201114-6c61c88383e4/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= +github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/pat v0.0.0-20180118222023-199c85a7f6d1/go.mod h1:YeAe0gNeiNT5hoiZRI4yiOky6jVdNvfO2N6Kav/HmxY= +github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= +github.com/gorilla/sessions v1.1.2/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w= +github.com/gorilla/sessions v1.1.3/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/gregjones/httpcache v0.0.0-20181110185634-c63ab54fda8f/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= +github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI= +github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jackc/fake v0.0.0-20150926172116-812a484cc733/go.mod h1:WrMFNQdiFJ80sQsxDoMokWK1W5TQtxBFNpzWTD84ibQ= +github.com/jackc/pgx v3.2.0+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= +github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU= +github.com/joho/godotenv v1.2.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/karrick/godirwalk v1.7.5/go.mod h1:2c9FRhkDxdIbgkOnCEvnSWs71Bhugbl46shStcFDJ34= +github.com/karrick/godirwalk v1.7.7/go.mod h1:2c9FRhkDxdIbgkOnCEvnSWs71Bhugbl46shStcFDJ34= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v0.0.0-20180402223658-b729f2633dfe/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 h1:2gxZ0XQIU/5z3Z3bUBu+FXuk2pFbkN6tcwi/pjyaDic= +github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/markbates/deplist v1.0.4/go.mod h1:gRRbPbbuA8TmMiRvaOzUlRfzfjeCCBqX2A6arxN01MM= +github.com/markbates/deplist v1.0.5/go.mod h1:gRRbPbbuA8TmMiRvaOzUlRfzfjeCCBqX2A6arxN01MM= +github.com/markbates/going v1.0.2/go.mod h1:UWCk3zm0UKefHZ7l8BNqi26UyiEMniznk8naLdTcy6c= +github.com/markbates/grift v1.0.4/go.mod h1:wbmtW74veyx+cgfwFhlnnMWqhoz55rnHR47oMXzsyVs= +github.com/markbates/hmax v1.0.0/go.mod h1:cOkR9dktiESxIMu+65oc/r/bdY4bE8zZw3OLhLx0X2c= +github.com/markbates/inflect v1.0.0/go.mod h1:oTeZL2KHA7CUX6X+fovmK9OvIOFuqu0TwdQrZjLTh88= +github.com/markbates/inflect v1.0.1/go.mod h1:uv3UVNBe5qBIfCm8O8Q+DW+S1EopeyINj+Ikhc7rnCk= +github.com/markbates/inflect v1.0.3/go.mod h1:1fR9+pO2KHEO9ZRtto13gDwwZaAKstQzferVeWqbgNs= +github.com/markbates/inflect v1.0.4/go.mod h1:1fR9+pO2KHEO9ZRtto13gDwwZaAKstQzferVeWqbgNs= +github.com/markbates/oncer v0.0.0-20180924031910-e862a676800b/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= +github.com/markbates/oncer v0.0.0-20180924034138-723ad0170a46/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= +github.com/markbates/oncer v0.0.0-20181014194634-05fccaae8fc4/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= +github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= +github.com/markbates/refresh v1.4.10/go.mod h1:NDPHvotuZmTmesXxr95C9bjlw1/0frJwtME2dzcVKhc= +github.com/markbates/safe v1.0.0/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= +github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= +github.com/markbates/sigtx v1.0.0/go.mod h1:QF1Hv6Ic6Ca6W+T+DL0Y/ypborFKyvUY9HmuCD4VeTc= +github.com/markbates/willie v1.0.9/go.mod h1:fsrFVWl91+gXpx/6dv715j7i11fYPfZ9ZGfH0DQzY7w= +github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a/go.mod h1:M1qoD/MqPgTZIk0EWKB38wE28ACRfVcn+cU08jyArI0= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.0.0/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/monoculum/formam v0.0.0-20180901015400-4e68be1d79ba/go.mod h1:RKgILGEJq24YyJ2ban8EO0RUVSJlF1pGsEvoLEACr/Q= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d h1:7PxY7LVfSZm7PEeBTyK1rj1gABdCO2mbri6GKO1cMDs= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.4.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.3.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= +github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/petar/GoLLRB v0.0.0-20130427215148-53be0d36a84c/go.mod h1:HUpKUBZnpzkdx0kD/+Yfuft+uD3zHGtXF/XJB14TUr4= +github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= +github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190109181635-f287a105a20e/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.1.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190104112138-b1a0a9a36d74/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= +github.com/rogpeppe/go-internal v1.0.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/serenize/snaker v0.0.0-20171204205717-a683aaf2d516/go.mod h1:Yow6lPLSAXx2ifx470yD/nUe22Dv5vBvxK/UK9UUTVs= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= +github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= +github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU= +github.com/shurcooL/highlight_go v0.0.0-20170515013102-78fb10f4a5f8/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag= +github.com/shurcooL/octicon v0.0.0-20180602230221-c42b0e3b24d9/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ= +github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/sirupsen/logrus v1.1.0/go.mod h1:zrgwTnHtNr00buQ1vSptGe8m1f/BbgsPukg8qsT7A+A= +github.com/sirupsen/logrus v1.1.1/go.mod h1:zrgwTnHtNr00buQ1vSptGe8m1f/BbgsPukg8qsT7A+A= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= +github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.0/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/viper v1.2.1/go.mod h1:P4AexN0a+C9tGAnUFNwDMYYZv3pjFuvmeiMyKRaNVlI= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/unrolled/secure v0.0.0-20180918153822-f340ee86eb8b/go.mod h1:mnPT77IAdsi/kV7+Es7y+pXALeV3h7G6dQF6mNYjcLA= +github.com/unrolled/secure v0.0.0-20181005190816-ff9db2ff917f/go.mod h1:mnPT77IAdsi/kV7+Es7y+pXALeV3h7G6dQF6mNYjcLA= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.2.0 h1:6I+W7f5VwC5SV9dNrZ3qXrDB9mD0dyGOi/ZJmYw03T4= +go.uber.org/multierr v1.2.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181001203147-e3636079e1a4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181015023909-0c41d7ab0a0e/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181024171144-74cb1d3d52f4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181025113841-85e1b3f9139a/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181025213731-e84da0312774/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181106171534-e4dc69e5b2fd/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181112202954-3d3f9f413869/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181127143415-eb0de9b17e85/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180112015858-5ccada7d0a7b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180816102801-aaf60122140d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180921000356-2f5d2388922f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180926154720-4dfa2610cdf3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181017193950-04a2e542c03f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181102091132-c10e9556a7bc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181207154023-610586996380/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190110200230-915654e7eabc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190328230028-74de082e2cca h1:hyA6yiAgbUwuWqtscNvWAI7U1CtlaD1KilQ6iudt1aI= +golang.org/x/net v0.0.0-20190328230028-74de082e2cca/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190110195249-fd3eaa146cbb/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180117170059-2c42eef0765b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180816055513-1c9583448a9c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180906133057-8cf3aee42992/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180921163948-d47a0f339242/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180927150500-dad3d9fb7b6e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181005133103-4497e2df6f9e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181011152604-fa43e7bc11ba/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181022134430-8a28ead16f52/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181024145615-5cd93ef61a7c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181025063200-d989b31c8746/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026064943-731415f00dce/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181106135930-3a76605856fd/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181206074257-70b957f3b65e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190109145017-48ac38b7c8cb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313 h1:pczuHS43Cp2ktBEEmLwScxgjWsBSzdaQiKzUyf3DTTc= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20171227012246-e19ae1496984/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20161028155119-f51c12702a4d/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181003024731-2f84ea8ef872/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181006002542-f60d9635b16a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181008205924-a2b3f7f249e9/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181010214653-38981630ecb4/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181013182035-5e66757b835f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181017214349-06f26fdaaa28/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181024171208-a2dc47679d30/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181026183834-f60e5f99f081/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181105230042-78dc5bac0cac/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181107215632-34b416bd17b3/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181114190951-94339b83286c/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181119130350-139d099f6620/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181127195227-b4e97c0ed882/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181127232545-e782529d0ddd/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181205224935-3576414c54a4/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181206194817-bcd4e47d0288/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190110211028-68c5ac90f574/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac h1:MQEvx39qSf8vyrx3XRaOe+j1UDIzKwkYOVObRgGPVqI= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +gomodules.xyz/jsonpatch/v2 v2.0.0 h1:lHNQverf0+Gm1TbSbVIDWVXOhZ2FpZopxRqpr2uIjs4= +gomodules.xyz/jsonpatch/v2 v2.0.0/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= +gomodules.xyz/jsonpatch/v2 v2.0.1 h1:xyiBuvkD2g5n7cYzx6u2sxQvsAy4QJsZFCzGVdzOXZ0= +gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= +gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19 h1:Lj2SnHtxkRGJDqnGaSjo+CCdIieEnwVazbOXILwQemk= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= +gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/mail.v2 v2.0.0-20180731213649-a0242b2233b4/go.mod h1:htwXN1Qh09vZJ1NVKxQqHPBaCBbzKhp5GzuJEA4VJWw= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.0.0/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +k8s.io/api v0.0.0-20190222213804-5cb15d344471 h1:MzQGt8qWQCR+39kbYRd0uQqsvSidpYqJLFeWiJ9l4OE= +k8s.io/api v0.0.0-20190222213804-5cb15d344471/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= +k8s.io/apiextensions-apiserver v0.0.0-20181126155829-0cd23ebeb688/go.mod h1:IxkesAMoaCRoLrPJdZNZUQp9NfZnzqaVzLhb2VEQzXE= +k8s.io/apiextensions-apiserver v0.0.0-20191026071228-81c2f4fbaa0d h1:GFxX2fe6Abf9DS0t/AZgzh+aZgXOo8LXGLvXrqwSuq4= +k8s.io/apiextensions-apiserver v0.0.0-20191026071228-81c2f4fbaa0d/go.mod h1:EKccLhOvqRM3ke5NY3Va72VYrZnMYjup+T4CxF+Sk5k= +k8s.io/apimachinery v0.0.0-20190221213512-86fb29eff628 h1:UYfHH+KEF88OTg+GojQUwFTNxbxwmoktLwutUzR0GPg= +k8s.io/apimachinery v0.0.0-20190221213512-86fb29eff628/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= +k8s.io/apiserver v0.0.0-20191026070530-d1b1b64dd924/go.mod h1:2icax/zfCfh218n2Nr4gHB9qYsb8ZrGngG1qkdHvww0= +k8s.io/client-go v0.0.0-20180806134042-1f13a808da65/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= +k8s.io/client-go v0.0.0-20181126152608-d082d5923d3c/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= +k8s.io/client-go v0.0.0-20190624085356-2c6e35a5b9cf/go.mod h1:rQMvHbaXi4pHSLf91Z1YI3h2Av+T3jsFKEIAWucw7hc= +k8s.io/client-go v0.0.0-20191026065934-0bdba2f91880/go.mod h1:ZLqcZX6t7degTAe8tELFpWE+ekFiCB7rH8Wc4u320BA= +k8s.io/client-go v10.0.0+incompatible h1:F1IqCqw7oMBzDkqlcBymRq1450wD0eNqLE9jzUrIi34= +k8s.io/client-go v10.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= +k8s.io/code-generator v0.0.0-20181009084210-b36598f30652/go.mod h1:MYiN+ZJZ9HkETbgVZdWw2AsuAi9PZ4V80cwfuf2axe8= +k8s.io/code-generator v0.0.0-20181206115026-3a2206dd6a78/go.mod h1:MYiN+ZJZ9HkETbgVZdWw2AsuAi9PZ4V80cwfuf2axe8= +k8s.io/code-generator v0.0.0-20190620073620-d55040311883 h1:NWWNvN6IdpmQvZ43rVccCI8GPUrheK8XNdqeKycw0DI= +k8s.io/code-generator v0.0.0-20190620073620-d55040311883/go.mod h1:+a+9g9W0llgbgvx6qOb+VbeZPH5km1FrVyMQe9/jkQY= +k8s.io/code-generator v0.0.0-20191026065352-f361089c127c h1:PPfeQeI0wxiHIlVH9Gg64UZs/64GnNoRKrSb3TFSssM= +k8s.io/code-generator v0.0.0-20191026065352-f361089c127c/go.mod h1:HtDEU3n5Xo1vbwjXWiJ/lFNb5r6BWBz6aZU1IZTr4eA= +k8s.io/component-base v0.0.0-20191026070247-e99719fced85/go.mod h1:rrj6eT+Plsc1xyn4qx2jt37WhDQeaZOqmHiSTwi5vAA= +k8s.io/gengo v0.0.0-20180813235010-4242d8e6c5db/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20181113154421-fd15ee9cc2f7/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20190822140433-26a664648505 h1:ZY6yclUKVbZ+SdWnkfY+Je5vrMpKOxmGeKRbsXVmqYM= +k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.1.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.1 h1:RVgyDHY/kFKtLqh67NvEWIgkMneNoIrdkN0CxDSQc68= +k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= +k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +k8s.io/kube-openapi v0.0.0-20181109181836-c59034cc13d5/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= +k8s.io/kube-openapi v0.0.0-20181114233023-0317810137be/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= +k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf h1:EYm5AW/UUDbnmnI+gK0TJDVK9qPLhM+sRHYanNKw0EQ= +k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= +k8s.io/sample-controller v0.0.0-20180926075748-b8a1e69103ae/go.mod h1:ulrg2qtVB4979TuA1BhaD/oGPZxO5NRWwHbk2FgaC3g= +k8s.io/sample-controller v0.0.0-20181221200518-c3a5aa93b2bf/go.mod h1:ulrg2qtVB4979TuA1BhaD/oGPZxO5NRWwHbk2FgaC3g= +k8s.io/sample-controller v0.0.0-20190625130054-294bc0f66822 h1:UXyvj/kuxD1b4Bq48X1TElUdQmYm6NbfC5wztvGY6pU= +k8s.io/sample-controller v0.0.0-20190625130054-294bc0f66822/go.mod h1:sUcuCC/zTFmUBinV320YPrdHD6HNxzVhJ/FVOBijn2c= +k8s.io/utils v0.0.0-20190221042446-c2654d5206da/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0= +k8s.io/utils v0.0.0-20191010214722-8d271d903fe4 h1:Gi+/O1saihwDqnlmC8Vhv1M5Sp4+rbOmK9TbsLn8ZEA= +k8s.io/utils v0.0.0-20191010214722-8d271d903fe4/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= +modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= +modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= +modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= +modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= +modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= +sigs.k8s.io/controller-runtime v0.1.4/go.mod h1:HFAYoOh6XMV+jKF1UjFwrknPbowfyHEHHRdJMf2jMX8= +sigs.k8s.io/controller-runtime v0.1.9/go.mod h1:HFAYoOh6XMV+jKF1UjFwrknPbowfyHEHHRdJMf2jMX8= +sigs.k8s.io/controller-runtime v0.1.12 h1:ovDq28E64PeY1yR+6H7DthakIC09soiDCrKvfP2tPYo= +sigs.k8s.io/controller-runtime v0.1.12/go.mod h1:HFAYoOh6XMV+jKF1UjFwrknPbowfyHEHHRdJMf2jMX8= +sigs.k8s.io/controller-tools v0.1.8/go.mod h1:6g08p9m9G/So3sBc1AOQifHfhxH/mb6Sc4z0LMI8XMw= +sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= +sigs.k8s.io/structured-merge-diff v0.0.0-20190817042607-6149e4549fca/go.mod h1:IIgPezJWb76P0hotTxzDbWsMYB8APh18qZnxkomBpxA= +sigs.k8s.io/testing_frameworks v0.1.1/go.mod h1:VVBKrHmJ6Ekkfz284YKhQePcdycOzNH9qL6ht1zEr/U= +sigs.k8s.io/testing_frameworks v0.1.2 h1:vK0+tvjF0BZ/RYFeZ1E6BYBwHJJXhjuZ3TdsEKH+UQM= +sigs.k8s.io/testing_frameworks v0.1.2/go.mod h1:ToQrwSC3s8Xf/lADdZp3Mktcql9CG0UAmdJG9th5i0w= +sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/hack/codegen.sh b/hack/codegen.sh index 8fd5fb3b..175a90de 100755 --- a/hack/codegen.sh +++ b/hack/codegen.sh @@ -1,5 +1,6 @@ +#!/usr/bin/env bash set -euo pipefail go generate ./pkg/... ./cmd/... -go run vendor/sigs.k8s.io/controller-tools/cmd/controller-gen/main.go all go fmt ./pkg/... ./cmd/... +# TODO goimports diff --git a/pipeline.sh b/pipeline.sh index 94ff7e88..57c02ff9 100755 --- a/pipeline.sh +++ b/pipeline.sh @@ -1,14 +1,13 @@ +#!/usr/bin/env bash set -euo pipefail -RELEASE="$1" +VERSION="$1" -echo "codegen" -hack/codegen.sh echo "test" test/test.sh echo "build" -build/build.sh +build/build.sh "$VERSION" echo "e2e test" -test/e2e/e2e.sh +test/e2e/e2e.sh "$VERSION" echo "release" -release/release.sh $RELEASE +release/release.sh "$VERSION" diff --git a/pkg/apis/apis.go b/pkg/apis/apis.go index 337fb821..c42444ae 100644 --- a/pkg/apis/apis.go +++ b/pkg/apis/apis.go @@ -15,7 +15,7 @@ limitations under the License. */ // Generate deepcopy for apis -//go:generate go run ../../vendor/k8s.io/code-generator/cmd/deepcopy-gen/main.go -O zz_generated.deepcopy -i ./... -h ../../hack/boilerplate.go.txt +//go:generate go run -mod vendor k8s.io/code-generator/cmd/deepcopy-gen/main.go -O zz_generated.deepcopy -i ./... -h ../../hack/boilerplate.go.txt // Package apis contains Kubernetes API groups. package apis diff --git a/pkg/apis/config/group.go b/pkg/apis/config/group.go new file mode 100644 index 00000000..d912156b --- /dev/null +++ b/pkg/apis/config/group.go @@ -0,0 +1 @@ +package config diff --git a/pkg/apis/config/v1alpha1/types.go b/pkg/apis/config/v1alpha1/types.go new file mode 100644 index 00000000..d53b2613 --- /dev/null +++ b/pkg/apis/config/v1alpha1/types.go @@ -0,0 +1,33 @@ +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type Agent struct { + metav1.TypeMeta `json:",inline"` + UseClusterNamespaces bool `json:"useClusterNamespaces"` + Remotes []Remote `json:"remotes"` +} + +type Remote struct { + Kubeconfig string `json:"kubeconfig"` + Context string `json:"context"` + ClusterName string `json:"clusterName"` +} + +type Scheduler struct { + metav1.TypeMeta `json:",inline"` + Clusters []Cluster `json:"clusters"` + UseClusterNamespaces bool `json:"useClusterNamespaces"` +} + +type Cluster struct { + Name string `json:"name"` + ClusterNamespace string `json:"clusterNamespace"` + Memberships []Membership `json:"memberships"` +} + +type Membership struct { + FederationName string `json:"federationName"` +} diff --git a/pkg/apis/multicluster/v1alpha1/nodeobservation_types_test.go b/pkg/apis/multicluster/v1alpha1/nodeobservation_types_test.go deleted file mode 100644 index bfa3131d..00000000 --- a/pkg/apis/multicluster/v1alpha1/nodeobservation_types_test.go +++ /dev/null @@ -1,58 +0,0 @@ -/* -Copyright 2018 The Multicluster-Scheduler Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha1 - -import ( - "testing" - - "github.com/onsi/gomega" - "golang.org/x/net/context" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" -) - -func TestStorageNodeObservation(t *testing.T) { - key := types.NamespacedName{ - Name: "foo", - Namespace: "default", - } - created := &NodeObservation{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "default", - }} - g := gomega.NewGomegaWithT(t) - - // Test Create - fetched := &NodeObservation{} - g.Expect(c.Create(context.TODO(), created)).NotTo(gomega.HaveOccurred()) - - g.Expect(c.Get(context.TODO(), key, fetched)).NotTo(gomega.HaveOccurred()) - g.Expect(fetched).To(gomega.Equal(created)) - - // Test Updating the Labels - updated := fetched.DeepCopy() - updated.Labels = map[string]string{"hello": "world"} - g.Expect(c.Update(context.TODO(), updated)).NotTo(gomega.HaveOccurred()) - - g.Expect(c.Get(context.TODO(), key, fetched)).NotTo(gomega.HaveOccurred()) - g.Expect(fetched).To(gomega.Equal(updated)) - - // Test Delete - g.Expect(c.Delete(context.TODO(), fetched)).NotTo(gomega.HaveOccurred()) - g.Expect(c.Get(context.TODO(), key, fetched)).To(gomega.HaveOccurred()) -} diff --git a/pkg/apis/multicluster/v1alpha1/nodepool_types_test.go b/pkg/apis/multicluster/v1alpha1/nodepool_types_test.go deleted file mode 100644 index 19b1835b..00000000 --- a/pkg/apis/multicluster/v1alpha1/nodepool_types_test.go +++ /dev/null @@ -1,56 +0,0 @@ -/* -Copyright 2018 The Multicluster-Scheduler Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha1 - -import ( - "testing" - - "github.com/onsi/gomega" - "golang.org/x/net/context" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" -) - -func TestStorageNodePool(t *testing.T) { - key := types.NamespacedName{ - Name: "foo", - } - created := &NodePool{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - }} - g := gomega.NewGomegaWithT(t) - - // Test Create - fetched := &NodePool{} - g.Expect(c.Create(context.TODO(), created)).NotTo(gomega.HaveOccurred()) - - g.Expect(c.Get(context.TODO(), key, fetched)).NotTo(gomega.HaveOccurred()) - g.Expect(fetched).To(gomega.Equal(created)) - - // Test Updating the Labels - updated := fetched.DeepCopy() - updated.Labels = map[string]string{"hello": "world"} - g.Expect(c.Update(context.TODO(), updated)).NotTo(gomega.HaveOccurred()) - - g.Expect(c.Get(context.TODO(), key, fetched)).NotTo(gomega.HaveOccurred()) - g.Expect(fetched).To(gomega.Equal(updated)) - - // Test Delete - g.Expect(c.Delete(context.TODO(), fetched)).NotTo(gomega.HaveOccurred()) - g.Expect(c.Get(context.TODO(), key, fetched)).To(gomega.HaveOccurred()) -} diff --git a/pkg/apis/multicluster/v1alpha1/nodepoolobservation_types_test.go b/pkg/apis/multicluster/v1alpha1/nodepoolobservation_types_test.go deleted file mode 100644 index e063428e..00000000 --- a/pkg/apis/multicluster/v1alpha1/nodepoolobservation_types_test.go +++ /dev/null @@ -1,58 +0,0 @@ -/* -Copyright 2018 The Multicluster-Scheduler Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha1 - -import ( - "testing" - - "github.com/onsi/gomega" - "golang.org/x/net/context" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" -) - -func TestStorageNodePoolObservation(t *testing.T) { - key := types.NamespacedName{ - Name: "foo", - Namespace: "default", - } - created := &NodePoolObservation{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "default", - }} - g := gomega.NewGomegaWithT(t) - - // Test Create - fetched := &NodePoolObservation{} - g.Expect(c.Create(context.TODO(), created)).NotTo(gomega.HaveOccurred()) - - g.Expect(c.Get(context.TODO(), key, fetched)).NotTo(gomega.HaveOccurred()) - g.Expect(fetched).To(gomega.Equal(created)) - - // Test Updating the Labels - updated := fetched.DeepCopy() - updated.Labels = map[string]string{"hello": "world"} - g.Expect(c.Update(context.TODO(), updated)).NotTo(gomega.HaveOccurred()) - - g.Expect(c.Get(context.TODO(), key, fetched)).NotTo(gomega.HaveOccurred()) - g.Expect(fetched).To(gomega.Equal(updated)) - - // Test Delete - g.Expect(c.Delete(context.TODO(), fetched)).NotTo(gomega.HaveOccurred()) - g.Expect(c.Get(context.TODO(), key, fetched)).To(gomega.HaveOccurred()) -} diff --git a/pkg/apis/multicluster/v1alpha1/podobservation_types.go b/pkg/apis/multicluster/v1alpha1/podobservation_types.go index 258fe23a..2712e194 100644 --- a/pkg/apis/multicluster/v1alpha1/podobservation_types.go +++ b/pkg/apis/multicluster/v1alpha1/podobservation_types.go @@ -29,6 +29,8 @@ type PodObservationSpec struct { type PodObservationStatus struct { // +optional LiveState *corev1.Pod `json:"liveState,omitempty"` + // +optional + DelegateState *corev1.Pod `json:"delegateState,omitempty"` } // +genclient diff --git a/pkg/apis/multicluster/v1alpha1/podobservation_types_test.go b/pkg/apis/multicluster/v1alpha1/podobservation_types_test.go deleted file mode 100644 index 45bfe160..00000000 --- a/pkg/apis/multicluster/v1alpha1/podobservation_types_test.go +++ /dev/null @@ -1,58 +0,0 @@ -/* -Copyright 2018 The Multicluster-Scheduler Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha1 - -import ( - "testing" - - "github.com/onsi/gomega" - "golang.org/x/net/context" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" -) - -func TestStoragePodObservation(t *testing.T) { - key := types.NamespacedName{ - Name: "foo", - Namespace: "default", - } - created := &PodObservation{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "default", - }} - g := gomega.NewGomegaWithT(t) - - // Test Create - fetched := &PodObservation{} - g.Expect(c.Create(context.TODO(), created)).NotTo(gomega.HaveOccurred()) - - g.Expect(c.Get(context.TODO(), key, fetched)).NotTo(gomega.HaveOccurred()) - g.Expect(fetched).To(gomega.Equal(created)) - - // Test Updating the Labels - updated := fetched.DeepCopy() - updated.Labels = map[string]string{"hello": "world"} - g.Expect(c.Update(context.TODO(), updated)).NotTo(gomega.HaveOccurred()) - - g.Expect(c.Get(context.TODO(), key, fetched)).NotTo(gomega.HaveOccurred()) - g.Expect(fetched).To(gomega.Equal(updated)) - - // Test Delete - g.Expect(c.Delete(context.TODO(), fetched)).NotTo(gomega.HaveOccurred()) - g.Expect(c.Get(context.TODO(), key, fetched)).To(gomega.HaveOccurred()) -} diff --git a/pkg/apis/multicluster/v1alpha1/serviceobservation_types_test.go b/pkg/apis/multicluster/v1alpha1/serviceobservation_types_test.go deleted file mode 100644 index f522459a..00000000 --- a/pkg/apis/multicluster/v1alpha1/serviceobservation_types_test.go +++ /dev/null @@ -1,58 +0,0 @@ -/* -Copyright 2019 The Multicluster-Scheduler Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha1 - -import ( - "testing" - - "github.com/onsi/gomega" - "golang.org/x/net/context" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" -) - -func TestStorageServiceObservation(t *testing.T) { - key := types.NamespacedName{ - Name: "foo", - Namespace: "default", - } - created := &ServiceObservation{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "default", - }} - g := gomega.NewGomegaWithT(t) - - // Test Create - fetched := &ServiceObservation{} - g.Expect(c.Create(context.TODO(), created)).NotTo(gomega.HaveOccurred()) - - g.Expect(c.Get(context.TODO(), key, fetched)).NotTo(gomega.HaveOccurred()) - g.Expect(fetched).To(gomega.Equal(created)) - - // Test Updating the Labels - updated := fetched.DeepCopy() - updated.Labels = map[string]string{"hello": "world"} - g.Expect(c.Update(context.TODO(), updated)).NotTo(gomega.HaveOccurred()) - - g.Expect(c.Get(context.TODO(), key, fetched)).NotTo(gomega.HaveOccurred()) - g.Expect(fetched).To(gomega.Equal(updated)) - - // Test Delete - g.Expect(c.Delete(context.TODO(), fetched)).NotTo(gomega.HaveOccurred()) - g.Expect(c.Get(context.TODO(), key, fetched)).To(gomega.HaveOccurred()) -} diff --git a/pkg/apis/multicluster/v1alpha1/v1alpha1_suite_test.go b/pkg/apis/multicluster/v1alpha1/v1alpha1_suite_test.go deleted file mode 100644 index bd2e7f9f..00000000 --- a/pkg/apis/multicluster/v1alpha1/v1alpha1_suite_test.go +++ /dev/null @@ -1,55 +0,0 @@ -/* -Copyright 2018 The Multicluster-Scheduler Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha1 - -import ( - "log" - "os" - "path/filepath" - "testing" - - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/envtest" -) - -var cfg *rest.Config -var c client.Client - -func TestMain(m *testing.M) { - t := &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "..", "config", "crds")}, - } - - err := SchemeBuilder.AddToScheme(scheme.Scheme) - if err != nil { - log.Fatal(err) - } - - if cfg, err = t.Start(); err != nil { - log.Fatal(err) - } - - if c, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}); err != nil { - log.Fatal(err) - } - - code := m.Run() - t.Stop() - os.Exit(code) -} diff --git a/pkg/common/constants.go b/pkg/common/constants.go index 083419ea..9e2af071 100644 --- a/pkg/common/constants.go +++ b/pkg/common/constants.go @@ -16,30 +16,30 @@ limitations under the License. package common -// TODO... standardize label and annotation propagation story - var ( KeyPrefix = "multicluster.admiralty.io/" - AnnotationKeyElect = KeyPrefix + "elect" - AnnotationKeyClusterName = KeyPrefix + "clustername" - AnnotationKeyServiceDependenciesSelector = KeyPrefix + "service-dependencies-selector" + // annotations on source pod (by user) and proxy pods (copied by mutating admission webhook) + + AnnotationKeyElect = KeyPrefix + "elect" + AnnotationKeyFederationName = KeyPrefix + "federationname" + AnnotationKeyClusterName = KeyPrefix + "clustername" + + // annotations on proxy pods (by mutating admission webhook) KeyPrefixSourcePod = KeyPrefix + "sourcepod-" AnnotationKeySourcePodManifest = KeyPrefixSourcePod + "manifest" - KeyPrefixProxyPod = KeyPrefix + "proxypod-" + // labels on delegate pods (by bind controller) - AnnotationKeyProxyPodClusterName = KeyPrefixProxyPod + "clustername" - AnnotationKeyProxyPodNamespace = KeyPrefixProxyPod + "namespace" - AnnotationKeyProxyPodName = KeyPrefixProxyPod + "name" + KeyPrefixProxyPod = KeyPrefix + "proxypod-" - KeyPrefixOriginal = KeyPrefix + "original-" + LabelKeyProxyPodClusterName = KeyPrefixProxyPod + "clustername" + LabelKeyProxyPodNamespace = KeyPrefixProxyPod + "namespace" + LabelKeyProxyPodName = KeyPrefixProxyPod + "name" - LabelKeyOriginalName = KeyPrefixOriginal + "name" - LabelKeyOriginalNamespace = KeyPrefixOriginal + "namespace" - LabelKeyOriginalClusterName = KeyPrefixOriginal + "clusterName" + // labels on delegate services (by global service controller) LabelKeyIsDelegate = KeyPrefix + "is-delegate" ) diff --git a/pkg/config/agent/agent.go b/pkg/config/agent/agent.go new file mode 100644 index 00000000..573edaa4 --- /dev/null +++ b/pkg/config/agent/agent.go @@ -0,0 +1,48 @@ +package agent + +import ( + configv1alpha1 "admiralty.io/multicluster-scheduler/pkg/apis/config/v1alpha1" + "admiralty.io/multicluster-service-account/pkg/config" + "flag" + "io/ioutil" + "k8s.io/client-go/rest" + "log" + "sigs.k8s.io/yaml" +) + +type Config struct { + Remotes []Remote +} + +type Remote struct { + ClientConfig *rest.Config + Namespace string + ClusterName string +} + +func New() Config { + agentCfg := Config{} + cfgPath := flag.String("config", "/etc/admiralty/config", "") + s, err := ioutil.ReadFile(*cfgPath) + if err != nil { + log.Fatalf("cannot open agent configuration: %v", err) + } + raw := &configv1alpha1.Agent{} + if err := yaml.Unmarshal(s, raw); err != nil { + log.Fatalf("cannot unmarshal agent configuration: %v", err) + } + for _, m := range raw.Remotes { + cfg, ns, err := config.ConfigAndNamespaceForKubeconfigAndContext(m.Kubeconfig, m.Context) + if err != nil { + log.Fatalf("cannot load kubeconfig: %v", err) + } + r := Remote{ClientConfig: cfg, Namespace: ns} + if raw.UseClusterNamespaces { + r.ClusterName = r.Namespace + } else { + r.ClusterName = m.ClusterName + } + agentCfg.Remotes = append(agentCfg.Remotes, r) + } + return agentCfg +} diff --git a/pkg/config/scheduler/scheduler.go b/pkg/config/scheduler/scheduler.go new file mode 100644 index 00000000..15ee35bf --- /dev/null +++ b/pkg/config/scheduler/scheduler.go @@ -0,0 +1,150 @@ +package scheduler + +import ( + "flag" + "io/ioutil" + "log" + + "admiralty.io/multicluster-controller/pkg/patterns/gc" + configv1alpha1 "admiralty.io/multicluster-scheduler/pkg/apis/config/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/yaml" +) + +// TODO... better typing + +type Config struct { + // Namespaces to watch + Namespaces []string + + // NamespaceForCluster to create decisions into + NamespaceForCluster map[string]string + + // FederationsByCluster for now unused outside of transform + FederationsByCluster map[string]map[string]struct{} + + // ClustersByFederation to filter observations in schedule when federation annotation is NOT empty + ClustersByFederation map[string]map[string]struct{} + + // NamespacesByFederation to list observations in schedule when federation annotation is NOT empty + NamespacesByFederation map[string][]string + + // PairedClustersByCluster to filter observations in schedule when federation annotation is empty (any federation) + PairedClustersByCluster map[string]map[string]struct{} + + // PairedNamespacesByCluster to list observations in schedule when federation annotation is empty (any federation) + PairedNamespacesByCluster map[string][]string + + // UseClusterNamespaces to determine whether source cluster names are the names of the observations's namespaces + // or the ParentClusterName multi-cluster GC label (trust the agent then). + UseClusterNamespaces bool +} + +func Load(schedulerNamespace string) *Config { + path := flag.String("config", "/etc/admiralty/config", "") + s, err := ioutil.ReadFile(*path) + if err != nil { + log.Fatalf("cannot open scheduler configuration: %v", err) + } + raw := &configv1alpha1.Scheduler{} + if err := yaml.Unmarshal(s, raw); err != nil { + log.Fatalf("cannot unmarshal scheduler configuration: %v", err) + } + return New(raw, schedulerNamespace) +} + +func New(raw *configv1alpha1.Scheduler, schedulerNamespace string) *Config { + setDefaults(raw, schedulerNamespace) + return transform(raw) +} + +func setDefaults(raw *configv1alpha1.Scheduler, schedulerNamespace string) { + for i := range raw.Clusters { + c := &raw.Clusters[i] + if c.ClusterNamespace == "" { + c.ClusterNamespace = schedulerNamespace + } + if len(c.Memberships) == 0 { + c.Memberships = []configv1alpha1.Membership{{FederationName: "default"}} + } + } +} + +func transform(raw *configv1alpha1.Scheduler) *Config { + cfg := &Config{ + NamespaceForCluster: map[string]string{}, + FederationsByCluster: map[string]map[string]struct{}{}, + ClustersByFederation: map[string]map[string]struct{}{}, + NamespacesByFederation: map[string][]string{}, + PairedClustersByCluster: map[string]map[string]struct{}{}, + PairedNamespacesByCluster: map[string][]string{}, + UseClusterNamespaces: raw.UseClusterNamespaces, + } + + namespaces := map[string]struct{}{} // intermediate var to dedup namespaces + //clustersByFed := map[string]map[string]struct{}{} // intermediate var to dedup clusters by fed + + for _, c := range raw.Clusters { + cfg.NamespaceForCluster[c.Name] = c.ClusterNamespace + namespaces[c.ClusterNamespace] = struct{}{} + + for _, m := range c.Memberships { + if cfg.FederationsByCluster[c.Name] == nil { + cfg.FederationsByCluster[c.Name] = map[string]struct{}{} + } + cfg.FederationsByCluster[c.Name][m.FederationName] = struct{}{} + + if cfg.ClustersByFederation[m.FederationName] == nil { + cfg.ClustersByFederation[m.FederationName] = map[string]struct{}{} + } + cfg.ClustersByFederation[m.FederationName][c.Name] = struct{}{} + } + } + + for ns := range namespaces { + cfg.Namespaces = append(cfg.Namespaces, ns) + } + //for f, cm := range clustersByFed { + // var cl []string + // for c, _ := range cm { + // cl = append(cl, c) + // } + // cfg.ClustersByFederation[f] = cl + //} + + for f, cs := range cfg.ClustersByFederation { + namespaces := map[string]struct{}{} // intermediate var to dedup namespaces + for c := range cs { + namespaces[cfg.NamespaceForCluster[c]] = struct{}{} + } + for ns := range namespaces { + cfg.NamespacesByFederation[f] = append(cfg.NamespacesByFederation[f], ns) + } + } + + for srcC, fs := range cfg.FederationsByCluster { + namespaces := map[string]struct{}{} // intermediate var to dedup namespaces + if cfg.PairedClustersByCluster[srcC] == nil { + cfg.PairedClustersByCluster[srcC] = map[string]struct{}{} + } + for f := range fs { + for c := range cfg.ClustersByFederation[f] { + cfg.PairedClustersByCluster[srcC][c] = struct{}{} + namespaces[cfg.NamespaceForCluster[c]] = struct{}{} + } + } + for ns := range namespaces { + cfg.PairedNamespacesByCluster[srcC] = append(cfg.PairedNamespacesByCluster[srcC], ns) + } + } + + return cfg +} + +func (c *Config) GetObservationClusterName(obs v1.Object) string { + if c.UseClusterNamespaces { + return obs.GetNamespace() + } else { + return obs.GetLabels()[gc.LabelParentClusterName] + } +} diff --git a/pkg/config/scheduler/scheduler_test.go b/pkg/config/scheduler/scheduler_test.go new file mode 100644 index 00000000..0d8308ef --- /dev/null +++ b/pkg/config/scheduler/scheduler_test.go @@ -0,0 +1,32 @@ +package scheduler + +import ( + "testing" + + configv1alpha1 "admiralty.io/multicluster-scheduler/pkg/apis/config/v1alpha1" + "github.com/go-test/deep" +) + +func TestNew(t *testing.T) { + raw := &configv1alpha1.Scheduler{ + Clusters: []configv1alpha1.Cluster{{ + Name: "c1", + }, { + Name: "c2", + }}, + } + cfg := New(raw, "default") + expected := &Config{ + Namespaces: []string{"default"}, + NamespaceForCluster: map[string]string{"c1": "default", "c2": "default"}, + FederationsByCluster: map[string]map[string]struct{}{"c1": {"default": {}}, "c2": {"default": {}}}, + ClustersByFederation: map[string]map[string]struct{}{"default": {"c1": {}, "c2": {}}}, + NamespacesByFederation: map[string][]string{"default": {"default"}}, + PairedClustersByCluster: map[string]map[string]struct{}{"c1": {"c1": {}, "c2": {}}, "c2": {"c1": {}, "c2": {}}}, + PairedNamespacesByCluster: map[string][]string{"c1": {"default"}, "c2": {"default"}}, + UseClusterNamespaces: false, + } + if diff := deep.Equal(cfg, expected); diff != nil { + t.Errorf("diff: %v", diff) + } +} diff --git a/pkg/controllers/bind/bind.go b/pkg/controllers/bind/bind.go new file mode 100644 index 00000000..66f52901 --- /dev/null +++ b/pkg/controllers/bind/bind.go @@ -0,0 +1,189 @@ +/* +Copyright 2018 The Multicluster-Scheduler Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bind + +import ( + "fmt" + "strings" + + "admiralty.io/multicluster-controller/pkg/cluster" + "admiralty.io/multicluster-controller/pkg/controller" + "admiralty.io/multicluster-controller/pkg/patterns/gc" + "admiralty.io/multicluster-scheduler/pkg/apis/multicluster/v1alpha1" + "admiralty.io/multicluster-scheduler/pkg/common" + schedulerconfig "admiralty.io/multicluster-scheduler/pkg/config/scheduler" + "github.com/ghodss/yaml" + "github.com/go-test/deep" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func NewController(c *cluster.Cluster, schedCfg *schedulerconfig.Config) (*controller.Controller, error) { + // we can optimize GC's getChild if we know that children can only be in one namespace + // (there could actually be a further optimization since we know which cluster, hence which namespace, when we list) + // besides, RBAC requires a namespaced list when not in clusterNamespace mode + childNamespace := "" + if len(schedCfg.Namespaces) == 1 { + childNamespace = schedCfg.Namespaces[0] + } + return gc.NewController(c, c, gc.Options{ + ParentPrototype: &v1alpha1.PodObservation{}, + ChildPrototype: &v1alpha1.PodDecision{}, + ParentWatchOptions: controller.WatchOptions{ + Namespaces: schedCfg.Namespaces, + CustomPredicate: func(obj interface{}) bool { + podObs := obj.(*v1alpha1.PodObservation) + if podObs.Status.LiveState.Annotations == nil { + return false + } + _, isProxy := podObs.Status.LiveState.Annotations[common.AnnotationKeyElect] + clusterName, isScheduled := podObs.Status.LiveState.Annotations[common.AnnotationKeyClusterName] + srcClusterName := schedCfg.GetObservationClusterName(podObs) + _, isAllowed := schedCfg.PairedClustersByCluster[srcClusterName][clusterName] + return isProxy && isScheduled && isAllowed + }, + }, + ChildNamespace: childNamespace, + Applier: applier{schedCfg: schedCfg}, + MakeExpectedChildWhenFound: true, + }) +} + +type applier struct { + schedCfg *schedulerconfig.Config +} + +var _ gc.Applier = applier{} + +func (a applier) MakeChild(parent interface{}, expectedChild interface{}) error { + proxyPodObs := parent.(*v1alpha1.PodObservation) + delegatePodDec := expectedChild.(*v1alpha1.PodDecision) + + delegatePod, err := a.makeDelegatePod(proxyPodObs) + if err != nil { + return err + } + + delegatePodDec.Spec.Template.ObjectMeta = delegatePod.ObjectMeta + delegatePodDec.Spec.Template.Spec = delegatePod.Spec + + clusterName := proxyPodObs.Status.LiveState.Annotations[common.AnnotationKeyClusterName] + delegatePodDec.Namespace = a.schedCfg.NamespaceForCluster[clusterName] + delegatePodDec.Annotations = map[string]string{common.AnnotationKeyClusterName: clusterName} + return nil +} + +func (a applier) ChildNeedsUpdate(_ interface{}, child interface{}, expectedChild interface{}) (bool, error) { + delegatePodDec := child.(*v1alpha1.PodDecision) + expectedDelegatePodDec := expectedChild.(*v1alpha1.PodDecision) + + if diff := deep.Equal(delegatePodDec.Spec.Template.ObjectMeta, expectedDelegatePodDec.Spec.Template.ObjectMeta); diff != nil { + return true, nil + } + if diff := deep.Equal(delegatePodDec.Spec.Template.Spec, expectedDelegatePodDec.Spec.Template.Spec); diff != nil { + return true, nil + } + return false, nil +} + +func (a applier) MutateChild(_ interface{}, child interface{}, expectedChild interface{}) error { + delegatePodDec := child.(*v1alpha1.PodDecision) + expectedDelegatePodDec := expectedChild.(*v1alpha1.PodDecision) + + delegatePodDec.Spec.Template.ObjectMeta = expectedDelegatePodDec.Spec.Template.ObjectMeta + delegatePodDec.Spec.Template.Spec = expectedDelegatePodDec.Spec.Template.Spec + return nil +} + +func (a applier) makeDelegatePod(proxyPodObs *v1alpha1.PodObservation) (*corev1.Pod, error) { + proxyPod := proxyPodObs.Status.LiveState + srcPodManifest, ok := proxyPod.Annotations[common.AnnotationKeySourcePodManifest] + if !ok { + return nil, fmt.Errorf("no source pod manifest on proxy pod") + } + srcPod := &corev1.Pod{} + if err := yaml.Unmarshal([]byte(srcPodManifest), srcPod); err != nil { + return nil, fmt.Errorf("cannot unmarshal source pod manifest: %v", err) + } + + annotations := make(map[string]string) + for k, v := range srcPod.Annotations { + if !strings.HasPrefix(k, common.KeyPrefix) || k == common.AnnotationKeyFederationName { + // we don't want to mc-schedule the delegate pod with elect, + // and the target cluster name and source pod manifest are now redundant + // we only keep the federation name and user annotations + annotations[k] = v + } + } + + labels := make(map[string]string) + for k, v := range srcPod.Labels { + // we need to change the labels so as not to confuse potential controller of proxy pod, e.g., replica set + // if the original label key has a domain prefix, replace it with ours + // if it doesn't, add our domain prefix + // TODO: resolve conflict two keys have same name but different prefixes + // TODO: ensure we don't go over length limits + keySplit := strings.Split(k, "/") // note: assume no empty key (enforced by Kubernetes) + newKey := common.KeyPrefix + keySplit[len(keySplit)-1] + labels[newKey] = v + } + labels[common.LabelKeyProxyPodClusterName] = a.schedCfg.GetObservationClusterName(proxyPodObs) + labels[common.LabelKeyProxyPodNamespace] = proxyPod.Namespace + labels[common.LabelKeyProxyPodName] = proxyPod.Name + + delegatePod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: proxyPod.Namespace, // already defaults to "default" (vs. could be empty in srcPod) + Labels: labels, + Annotations: annotations}, + Spec: *srcPod.Spec.DeepCopy()} + + removeServiceAccount(delegatePod) + // TODO? add compatible fields instead of removing incompatible ones + // (for forward compatibility and we've certainly forgotten incompatible fields...) + // TODO... maybe make this configurable, sort of like Federation v2 Overrides + + return delegatePod, nil +} + +func removeServiceAccount(pod *corev1.Pod) { + var saSecretName string + for i, c := range pod.Spec.Containers { + j := -1 + for i, m := range c.VolumeMounts { + if m.MountPath == "/var/run/secrets/kubernetes.io/serviceaccount" { + saSecretName = m.Name + j = i + break + } + } + if j > -1 { + c.VolumeMounts = append(c.VolumeMounts[:j], c.VolumeMounts[j+1:]...) + pod.Spec.Containers[i] = c + } + } + j := -1 + for i, v := range pod.Spec.Volumes { + if v.Name == saSecretName { + j = i + break + } + } + if j > -1 { + pod.Spec.Volumes = append(pod.Spec.Volumes[:j], pod.Spec.Volumes[j+1:]...) + } +} diff --git a/pkg/controllers/delegatestate/delegatestate.go b/pkg/controllers/delegatestate/delegatestate.go new file mode 100644 index 00000000..6fc6c455 --- /dev/null +++ b/pkg/controllers/delegatestate/delegatestate.go @@ -0,0 +1,118 @@ +package delegatestate + +import ( + "context" + "fmt" + "reflect" + "time" + + "admiralty.io/multicluster-controller/pkg/cluster" + "admiralty.io/multicluster-controller/pkg/controller" + "admiralty.io/multicluster-controller/pkg/patterns" + "admiralty.io/multicluster-controller/pkg/patterns/gc" + "admiralty.io/multicluster-controller/pkg/reconcile" + "admiralty.io/multicluster-scheduler/pkg/apis/multicluster/v1alpha1" + "admiralty.io/multicluster-scheduler/pkg/common" + schedulerconfig "admiralty.io/multicluster-scheduler/pkg/config/scheduler" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func NewController(scheduler *cluster.Cluster, schedCfg *schedulerconfig.Config) (*controller.Controller, error) { + schedulerClient, err := scheduler.GetDelegatingClient() + if err != nil { + return nil, fmt.Errorf("getting delegating client: %v", err) + } + + co := controller.New(&reconciler{ + scheduler: schedulerClient, + schedCfg: schedCfg, + }, controller.Options{}) + + if err := co.WatchResourceReconcileObject(scheduler, &v1alpha1.PodObservation{}, controller.WatchOptions{ + CustomPredicate: func(obj interface{}) bool { + podObs := obj.(*v1alpha1.PodObservation) + proxyPodClusterName, isDelegate := podObs.Status.LiveState.Labels[common.LabelKeyProxyPodClusterName] + delegateClusterName := schedCfg.GetObservationClusterName(podObs) + _, isAllowed := schedCfg.PairedClustersByCluster[proxyPodClusterName][delegateClusterName] + // isAllowed prevents an attacker in an untrusted cluster from sending feedback via fake annotations + return isDelegate && isAllowed + }, + }); err != nil { + return nil, fmt.Errorf("setting up pod observation watch: %v", err) + } + + return co, nil +} + +type reconciler struct { + scheduler client.Client + schedCfg *schedulerconfig.Config +} + +func (r *reconciler) Reconcile(req reconcile.Request) (reconcile.Result, error) { + podObs := &v1alpha1.PodObservation{} + if err := r.scheduler.Get(context.Background(), req.NamespacedName, podObs); err != nil { + if !errors.IsNotFound(err) { + return reconcile.Result{}, fmt.Errorf("cannot get pod observation %s in namespace %s: %v", + req.Name, req.Namespace, err) + } + return reconcile.Result{}, nil + } + + pod := podObs.Status.LiveState + + proxyPodClusterName, ok := pod.Labels[common.LabelKeyProxyPodClusterName] + if !ok { + return reconcile.Result{}, fmt.Errorf("pod observation %s in namespace %s is missing label %s", + req.Name, req.Namespace, common.LabelKeyProxyPodClusterName) + } + proxyPodNs, ok := pod.Labels[common.LabelKeyProxyPodNamespace] + if !ok { + return reconcile.Result{}, fmt.Errorf("pod observation %s in namespace %s is missing label %s", + req.Name, req.Namespace, common.LabelKeyProxyPodNamespace) + } + proxyPodName, ok := pod.Labels[common.LabelKeyProxyPodName] + if !ok { + return reconcile.Result{}, fmt.Errorf("pod observation %s in namespace %s is missing label %s", + req.Name, req.Namespace, common.LabelKeyProxyPodName) + } + + proxyPodObsList := &v1alpha1.PodObservationList{} + s := labels.SelectorFromValidatedSet(labels.Set{ + gc.LabelParentClusterName: proxyPodClusterName, + gc.LabelParentNamespace: proxyPodNs, + gc.LabelParentName: proxyPodName, + }) + if err := r.scheduler.List(context.Background(), &client.ListOptions{ + Namespace: r.schedCfg.NamespaceForCluster[proxyPodClusterName], + LabelSelector: s, + }, proxyPodObsList); err != nil { + return reconcile.Result{}, fmt.Errorf("cannot list pod obs in namespace %s with label selector %s: %v", + proxyPodNs, s, err) + } + if len(proxyPodObsList.Items) == 0 { + return reconcile.Result{}, fmt.Errorf("proxy pod obs not found in namespace %s with label selector %s", + proxyPodNs, s) + } else if len(proxyPodObsList.Items) > 1 { + return reconcile.Result{}, fmt.Errorf("found duplicate proxy pod obs in namespace %s with label selector %s", + proxyPodNs, s) + } + proxyPodObs := &proxyPodObsList.Items[0] + + if !reflect.DeepEqual(proxyPodObs.Status.DelegateState, podObs.Status.LiveState) { + proxyPodObs.Status.DelegateState = podObs.Status.LiveState + if err := r.scheduler.Update(context.Background(), proxyPodObs); err != nil { + if patterns.IsOptimisticLockError(err) { + // TODO watch proxy pod observations instead, to requeue when the cache is updated + oneSec, _ := time.ParseDuration("1s") + return reconcile.Result{RequeueAfter: oneSec}, nil + } + return reconcile.Result{}, fmt.Errorf("cannot update proxy pod obs %s in namespace %s: %v", + proxyPodObs.Name, proxyPodObs.Namespace, err) + } + } + + return reconcile.Result{}, nil +} diff --git a/pkg/controllers/feedback/feedback.go b/pkg/controllers/feedback/feedback.go index 20bd5b0a..5affa028 100644 --- a/pkg/controllers/feedback/feedback.go +++ b/pkg/controllers/feedback/feedback.go @@ -22,15 +22,18 @@ import ( "fmt" "reflect" "strings" + "time" "admiralty.io/multicluster-controller/pkg/cluster" "admiralty.io/multicluster-controller/pkg/controller" + "admiralty.io/multicluster-controller/pkg/patterns" + "admiralty.io/multicluster-controller/pkg/patterns/gc" "admiralty.io/multicluster-controller/pkg/reconcile" - "admiralty.io/multicluster-scheduler/pkg/apis" "admiralty.io/multicluster-scheduler/pkg/apis/multicluster/v1alpha1" "admiralty.io/multicluster-scheduler/pkg/common" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -38,7 +41,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -func NewController(agent *cluster.Cluster, scheduler *cluster.Cluster, agentClientset *kubernetes.Clientset) (*controller.Controller, error) { +func NewController(agent *cluster.Cluster, scheduler *cluster.Cluster, observationNamespace string, + agentClientset *kubernetes.Clientset) (*controller.Controller, error) { + agentClient, err := agent.GetDelegatingClient() if err != nil { return nil, fmt.Errorf("getting delegating client for agent cluster: %v", err) @@ -51,18 +56,20 @@ func NewController(agent *cluster.Cluster, scheduler *cluster.Cluster, agentClie co := controller.New(&reconciler{ agent: agentClient, scheduler: schedulerClient, - agentContext: agent.Name, agentClientset: agentClientset, agentConfig: agent.Config, }, controller.Options{}) - if err := apis.AddToScheme(scheduler.GetScheme()); err != nil { - return nil, fmt.Errorf("adding APIs to scheduler cluster's scheme: %v", err) - } - if err := co.WatchResourceReconcileObject(scheduler, &v1alpha1.PodObservation{}, controller.WatchOptions{}); err != nil { + if err := co.WatchResourceReconcileObject(scheduler, &v1alpha1.PodObservation{}, controller.WatchOptions{ + Namespace: observationNamespace, + LabelSelector: labels.SelectorFromValidatedSet(labels.Set{gc.LabelParentClusterName: agent.GetClusterName()}), + CustomPredicate: func(obj interface{}) bool { + podObs := obj.(*v1alpha1.PodObservation) + return podObs.Status.DelegateState != nil + }, + }); err != nil { return nil, fmt.Errorf("setting up pod observation watch on scheduler cluster: %v", err) } - // TODO? watch proxy pod with custom handler return co, nil } @@ -70,46 +77,36 @@ func NewController(agent *cluster.Cluster, scheduler *cluster.Cluster, agentClie type reconciler struct { agent client.Client scheduler client.Client - agentContext string agentConfig *rest.Config agentClientset *kubernetes.Clientset } func (r *reconciler) Reconcile(req reconcile.Request) (reconcile.Result, error) { podObs := &v1alpha1.PodObservation{} - if err := r.scheduler.Get(context.TODO(), req.NamespacedName, podObs); err != nil { + if err := r.scheduler.Get(context.Background(), req.NamespacedName, podObs); err != nil { if !errors.IsNotFound(err) { return reconcile.Result{}, fmt.Errorf("cannot get pod observation %s in namespace %s in scheduler cluster: %v", req.Name, req.Namespace, err) } return reconcile.Result{}, nil } - delegatePod := podObs.Status.LiveState + delegatePod := podObs.Status.DelegateState // not nil thanks to watchoptions custompredicate - clusterName, ok := delegatePod.Annotations[common.AnnotationKeyProxyPodClusterName] + proxyPodNs, ok := podObs.Labels[gc.LabelParentNamespace] if !ok { - // not a multicluster pod - return reconcile.Result{}, nil - } - if clusterName != r.agentContext { - // request for other cluster, do nothing - // TODO: filter upstream (with Watch predicate) - return reconcile.Result{}, nil + return reconcile.Result{}, fmt.Errorf("pod observation %s in namespace %s in scheduler cluster is missing label %s", + req.Name, req.Namespace, gc.LabelParentNamespace) } - ns, ok := delegatePod.Annotations[common.AnnotationKeyProxyPodNamespace] + proxyPodName, ok := podObs.Labels[gc.LabelParentName] if !ok { - // not a multicluster pod - return reconcile.Result{}, nil - } - name, ok := delegatePod.Annotations[common.AnnotationKeyProxyPodName] - if !ok { - // not a multicluster pod - return reconcile.Result{}, nil + return reconcile.Result{}, fmt.Errorf("pod observation %s in namespace %s in scheduler cluster is missing label %s", + req.Name, req.Namespace, gc.LabelParentName) } + // TODO use controller ref instead? proxyPod := &corev1.Pod{} - if err := r.agent.Get(context.TODO(), types.NamespacedName{Namespace: ns, Name: name}, proxyPod); err != nil { + if err := r.agent.Get(context.Background(), types.NamespacedName{Namespace: proxyPodNs, Name: proxyPodName}, proxyPod); err != nil { if !errors.IsNotFound(err) { - return reconcile.Result{}, fmt.Errorf("cannot get proxy pod %s in namespace %s in agent cluster: %v", name, ns, err) + return reconcile.Result{}, fmt.Errorf("cannot get proxy pod %s in namespace %s in agent cluster: %v", proxyPodName, proxyPodNs, err) } return reconcile.Result{}, nil } @@ -122,7 +119,12 @@ func (r *reconciler) Reconcile(req reconcile.Request) (reconcile.Result, error) mcProxyPodAnnotations[k] = v } proxyPod.Annotations = mcProxyPodAnnotations - if err := r.agent.Update(context.TODO(), proxyPod); err != nil { + if err := r.agent.Update(context.Background(), proxyPod); err != nil { + if patterns.IsOptimisticLockError(err) { + // TODO watch proxy pods instead, to requeue when the cache is updated + oneSec, _ := time.ParseDuration("1s") + return reconcile.Result{RequeueAfter: oneSec}, nil + } return reconcile.Result{}, fmt.Errorf("cannot update proxy pod %s in namespace %s in agent cluster: %v", proxyPod.Name, proxyPod.Namespace, err) } } diff --git a/pkg/controllers/globalsvc/globalsvc.go b/pkg/controllers/globalsvc/globalsvc.go index 412c4fcd..7693a170 100644 --- a/pkg/controllers/globalsvc/globalsvc.go +++ b/pkg/controllers/globalsvc/globalsvc.go @@ -22,37 +22,45 @@ import ( "admiralty.io/multicluster-controller/pkg/cluster" "admiralty.io/multicluster-controller/pkg/controller" + "admiralty.io/multicluster-controller/pkg/patterns" + "admiralty.io/multicluster-controller/pkg/patterns/gc" "admiralty.io/multicluster-controller/pkg/reconcile" - "admiralty.io/multicluster-scheduler/pkg/apis" + "admiralty.io/multicluster-controller/pkg/reference" "admiralty.io/multicluster-scheduler/pkg/apis/multicluster/v1alpha1" "admiralty.io/multicluster-scheduler/pkg/common" + schedulerconfig "admiralty.io/multicluster-scheduler/pkg/config/scheduler" "github.com/go-test/deep" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) -func NewController(scheduler *cluster.Cluster) (*controller.Controller, error) { +func NewController(scheduler *cluster.Cluster, schedCfg *schedulerconfig.Config) (*controller.Controller, error) { client, err := scheduler.GetDelegatingClient() if err != nil { return nil, fmt.Errorf("getting delegating client for scheduler cluster: %v", err) } co := controller.New(&reconciler{ - client: client, - scheme: scheduler.GetScheme(), + client: client, + schedCfg: schedCfg, }, controller.Options{}) - if err := apis.AddToScheme(scheduler.GetScheme()); err != nil { - return nil, fmt.Errorf("adding APIs to scheduler cluster's scheme: %v", err) - } - if err := co.WatchResourceReconcileObject(scheduler, &v1alpha1.ServiceObservation{}, controller.WatchOptions{}); err != nil { + if err := co.WatchResourceReconcileObjectOverrideContext(scheduler, &v1alpha1.ServiceObservation{}, controller.WatchOptions{ + Namespaces: schedCfg.Namespaces, + CustomPredicate: func(obj interface{}) bool { + svcObs := obj.(*v1alpha1.ServiceObservation) + svc := svcObs.Status.LiveState + return svc.Annotations["io.cilium/global-service"] == "true" && + svc.Labels[common.LabelKeyIsDelegate] != "true" // no need to globalyze a delegate service (result of other service's globalyzation) + }, + }, ""); err != nil { return nil, fmt.Errorf("setting up proxy service observation watch: %v", err) } - if err := co.WatchResourceReconcileController(scheduler, &v1alpha1.ServiceDecision{}, controller.WatchOptions{}); err != nil { + if err := co.WatchResourceReconcileController(scheduler, &v1alpha1.ServiceDecision{}, controller.WatchOptions{ + Namespaces: schedCfg.Namespaces, + }); err != nil { return nil, fmt.Errorf("setting up delegate service decision watch: %v", err) } @@ -60,13 +68,13 @@ func NewController(scheduler *cluster.Cluster) (*controller.Controller, error) { } type reconciler struct { - client client.Client - scheme *runtime.Scheme + client client.Client + schedCfg *schedulerconfig.Config } func (r *reconciler) Reconcile(req reconcile.Request) (reconcile.Result, error) { svcObs := &v1alpha1.ServiceObservation{} - if err := r.client.Get(context.TODO(), req.NamespacedName, svcObs); err != nil { + if err := r.client.Get(context.Background(), req.NamespacedName, svcObs); err != nil { if !errors.IsNotFound(err) { return reconcile.Result{}, fmt.Errorf("cannot get service observation %s in namespace %s: %v", req.Name, req.Namespace, err) } @@ -76,59 +84,77 @@ func (r *reconciler) Reconcile(req reconcile.Request) (reconcile.Result, error) svc := svcObs.Status.LiveState - if svc.Labels[common.LabelKeyIsDelegate] == "true" { - // no need to globalyze a delegate service (result of other service's globalyzation) - return reconcile.Result{}, nil - } + srcClusterName := r.schedCfg.GetObservationClusterName(svcObs) - if svc.Annotations["io.cilium/global-service"] != "true" { - return reconcile.Result{}, nil + var clusters map[string]struct{} + fedName := svc.Annotations[common.AnnotationKeyFederationName] + if fedName == "" { + clusters = r.schedCfg.PairedClustersByCluster[srcClusterName] + } else { + clusters = r.schedCfg.ClustersByFederation[fedName] } - // HACK: get all cluster names from node pools - npObsL := &v1alpha1.NodePoolObservationList{} - if err := r.client.List(context.TODO(), &client.ListOptions{}, npObsL); err != nil { - return reconcile.Result{}, fmt.Errorf("cannot list node pool observations: %v", err) - } - clusterNames := make(map[string]struct{}) - for _, npObs := range npObsL.Items { - clusterNames[npObs.Status.LiveState.ClusterName] = struct{}{} - } - - for clusterName := range clusterNames { - if clusterName == svc.ClusterName { + for clusterName := range clusters { + if clusterName == srcClusterName { continue } - delSvc := makeDelegateService(svc, clusterName) - svcDecName := req.Name + "-" + clusterName + delSvc := makeDelegateService(svc) - svcDec := &v1alpha1.ServiceDecision{} - if err := r.client.Get(context.TODO(), types.NamespacedName{Name: svcDecName, Namespace: req.Namespace}, svcDec); err != nil { - if !errors.IsNotFound(err) { - return reconcile.Result{}, fmt.Errorf("cannot get service decision %s in namespace %s: %v", svcDecName, req.Namespace, err) - } + svcDecNamespace := r.schedCfg.NamespaceForCluster[clusterName] + l := &v1alpha1.ServiceDecisionList{} + s := labels.SelectorFromValidatedSet(labels.Set{ + gc.LabelParentName: svcObs.Name, + gc.LabelParentNamespace: svcObs.Namespace, + }) + err := r.client.List(context.Background(), &client.ListOptions{Namespace: svcDecNamespace, LabelSelector: s}, l) + if err != nil { + return reconcile.Result{}, fmt.Errorf( + "cannot list service decisions in namespace %s with label selector %s: %v", + svcDecNamespace, s, err) + } + if len(l.Items) > 1 { + return reconcile.Result{}, fmt.Errorf( + "duplicate service decisions found in namespace %s with label selector %s: %v", + svcDecNamespace, s, err) + } else if len(l.Items) == 0 { svcDec := &v1alpha1.ServiceDecision{} - svcDec.Name = svcDecName - svcDec.Namespace = req.Namespace + // we use generate name to avoid (unlikely) conflicts + genName := fmt.Sprintf("%s-%s-", svcObs.Namespace, svcObs.Name) + if len(genName) > 253 { + genName = genName[0:253] + } + svcDec.GenerateName = genName + svcDec.Namespace = svcDecNamespace + svcDec.Labels = labels.Set{ + gc.LabelParentName: svcObs.Name, + gc.LabelParentNamespace: svcObs.Namespace, + } + svcDec.Annotations = map[string]string{common.AnnotationKeyClusterName: clusterName} svcDec.Spec.Template.ObjectMeta = delSvc.ObjectMeta svcDec.Spec.Template.Spec = delSvc.Spec - if err := controllerutil.SetControllerReference(svcObs, svcDec, r.scheme); err != nil { - return reconcile.Result{}, fmt.Errorf("cannot set controller reference on service decision %s in namespace %s for owner %s in namespace %s: %v", - svcDecName, req.Namespace, svcObs.Name, svcObs.Namespace, err) + ref := reference.NewMulticlusterOwnerReference(svcObs, svc.GroupVersionKind(), "") + if err := reference.SetMulticlusterControllerReference(svcDec, ref); err != nil { + return reconcile.Result{}, fmt.Errorf( + "cannot set controller reference on service decision %s (name not yet generated) "+ + "in namespace %s for owner %s in namespace %s: %v", + genName, svcDecNamespace, svcObs.Name, svcObs.Namespace, err) } - if err := r.client.Create(context.TODO(), svcDec); err != nil { + if err := r.client.Create(context.Background(), svcDec); err != nil { if !errors.IsAlreadyExists(err) { - return reconcile.Result{}, fmt.Errorf("cannot create service decision %s in namespace %s: %v", svcDecName, req.Namespace, err) + return reconcile.Result{}, fmt.Errorf( + "cannot create service decision %s (name not yet generated) in namespace %s: %v", + genName, svcDecNamespace, err) } } continue } + svcDec := &l.Items[0] delSvc.Spec.ClusterIP = svcDec.Spec.Template.Spec.ClusterIP if deep.Equal(svcDec.Spec.Template.ObjectMeta, delSvc.ObjectMeta) == nil || @@ -139,18 +165,18 @@ func (r *reconciler) Reconcile(req reconcile.Request) (reconcile.Result, error) svcDec.Spec.Template.ObjectMeta = delSvc.ObjectMeta svcDec.Spec.Template.Spec = delSvc.Spec - if err := r.client.Update(context.TODO(), svcDec); err != nil { - return reconcile.Result{}, fmt.Errorf("cannot update delegate service decision %s in namespace %s: %v", svcDecName, req.Namespace, err) + if err := r.client.Update(context.Background(), svcDec); err != nil && !patterns.IsOptimisticLockError(err) { + return reconcile.Result{}, fmt.Errorf("cannot update delegate service decision %s in namespace %s: %v", + svcDec.Name, svcDec.Namespace, err) } } return reconcile.Result{}, nil } -func makeDelegateService(svc *corev1.Service, clusterName string) *corev1.Service { +func makeDelegateService(svc *corev1.Service) *corev1.Service { delSvc := &corev1.Service{} - delSvc.ClusterName = clusterName delSvc.Name = svc.Name delSvc.Namespace = svc.Namespace diff --git a/pkg/controllers/nodepool/nodepool.go b/pkg/controllers/nodepool/nodepool.go index 044de87c..80e5171a 100644 --- a/pkg/controllers/nodepool/nodepool.go +++ b/pkg/controllers/nodepool/nodepool.go @@ -23,8 +23,8 @@ import ( "admiralty.io/multicluster-controller/pkg/cluster" "admiralty.io/multicluster-controller/pkg/controller" + "admiralty.io/multicluster-controller/pkg/patterns" "admiralty.io/multicluster-controller/pkg/reconcile" - "admiralty.io/multicluster-scheduler/pkg/apis" "admiralty.io/multicluster-scheduler/pkg/apis/multicluster/v1alpha1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -42,23 +42,19 @@ var ( DefaultNodePool = "default" ) -func NewController(local *cluster.Cluster) (*controller.Controller, error) { - client, err := local.GetDelegatingClient() +func NewController(c *cluster.Cluster) (*controller.Controller, error) { + cl, err := c.GetDelegatingClient() if err != nil { - return nil, fmt.Errorf("getting delegating client for local cluster: %v", err) + return nil, fmt.Errorf("getting delegating client for cluster: %v", err) } - co := controller.New(&reconciler{client: client, scheme: local.GetScheme()}, controller.Options{}) + co := controller.New(&reconciler{client: cl, scheme: c.GetScheme()}, controller.Options{}) - if err := apis.AddToScheme(local.GetScheme()); err != nil { - return nil, fmt.Errorf("adding APIs to local cluster's scheme: %v", err) - } - if err := co.WatchResourceReconcileObject(local, &v1alpha1.NodePool{}, controller.WatchOptions{}); err != nil { + if err := co.WatchResourceReconcileObject(c, &v1alpha1.NodePool{}, controller.WatchOptions{}); err != nil { return nil, err } - // TODO: when multicluster-controller implements it, use WatchResource with arbitrary handler - h := &EnqueueRequestForNodePool{Context: local.Name, Queue: co.Queue} - if err := local.AddEventHandler(&corev1.Node{}, h); err != nil { + h := &EnqueueRequestForNodePool{Context: c.Name, Queue: co.Queue} + if err := co.WatchResource(c, &corev1.Node{}, h); err != nil { return nil, err } @@ -71,23 +67,20 @@ type reconciler struct { } func (r *reconciler) Reconcile(req reconcile.Request) (reconcile.Result, error) { - return reconcile.Result{}, r.reconcileNodePool(req.Name) -} - -func (r *reconciler) reconcileNodePool(name string) error { + name := req.Name nodes, err := r.getTaggedNodes(name) if err != nil { - return err + return reconcile.Result{}, err } onp := &v1alpha1.NodePool{} - if err := r.client.Get(context.TODO(), types.NamespacedName{Name: name}, onp); err != nil { + if err := r.client.Get(context.Background(), types.NamespacedName{Name: name}, onp); err != nil { if !errors.IsNotFound(err) { - return err + return reconcile.Result{}, err } // node pool not found if err := r.createNodePool(name, nodes); err != nil { - return err + return reconcile.Result{}, err } } else { // node pool exists @@ -95,12 +88,12 @@ func (r *reconciler) reconcileNodePool(name string) error { // find more nodes by selector moreNodes, err := r.getSelectedNodes(onp.Spec.Selector) if err != nil { - return err + return reconcile.Result{}, err } nodes = append(nodes, moreNodes...) - if err := r.updateNodePool(onp, nodes); err != nil { - return err + if err := r.updateNodePool(onp, nodes); err != nil && !patterns.IsOptimisticLockError(err) { + return reconcile.Result{}, err } } @@ -113,32 +106,32 @@ func (r *reconciler) reconcileNodePool(name string) error { n.Labels = map[string]string{} } n.Labels[NodePoolLabel] = name - if err := r.client.Update(context.TODO(), &n); err != nil { - return err + if err := r.client.Update(context.Background(), &n); err != nil && !patterns.IsOptimisticLockError(err) { + return reconcile.Result{}, err } } } - return nil + return reconcile.Result{}, nil } func (r *reconciler) getTaggedNodes(nodePoolName string) ([]corev1.Node, error) { var nodes []corev1.Node list := &corev1.NodeList{} - if err := r.client.List(context.TODO(), client.MatchingLabels(map[string]string{NodePoolLabel: nodePoolName}), list); err != nil { + if err := r.client.List(context.Background(), client.MatchingLabels(map[string]string{NodePoolLabel: nodePoolName}), list); err != nil { return nil, err } nodes = append(nodes, list.Items...) list = &corev1.NodeList{} - if err := r.client.List(context.TODO(), client.MatchingLabels(map[string]string{GKENodePoolLabel: nodePoolName}), list); err != nil { + if err := r.client.List(context.Background(), client.MatchingLabels(map[string]string{GKENodePoolLabel: nodePoolName}), list); err != nil { return nil, err } nodes = append(nodes, list.Items...) list = &corev1.NodeList{} - if err := r.client.List(context.TODO(), client.MatchingLabels(map[string]string{AKSNodePoolLabel: nodePoolName}), list); err != nil { + if err := r.client.List(context.Background(), client.MatchingLabels(map[string]string{AKSNodePoolLabel: nodePoolName}), list); err != nil { return nil, err } nodes = append(nodes, list.Items...) @@ -162,7 +155,7 @@ func (r *reconciler) getOrphanNodes() ([]corev1.Node, error) { if err := o.SetLabelSelector("!" + NodePoolLabel); err != nil { return nil, err } - if err := r.client.List(context.TODO(), o, list); err != nil { + if err := r.client.List(context.Background(), o, list); err != nil { return nil, err } nodes = append(nodes, list.Items...) @@ -183,7 +176,7 @@ func (r *reconciler) getSelectedNodes(selector *metav1.LabelSelector) ([]corev1. } o := &client.ListOptions{LabelSelector: selectorInterface} list := &corev1.NodeList{} - if err := r.client.List(context.TODO(), o, list); err != nil { + if err := r.client.List(context.Background(), o, list); err != nil { return nil, err } return list.Items, nil @@ -200,15 +193,15 @@ func (r *reconciler) createNodePool(name string, nodes []corev1.Node) error { NodeAllocatable: nodes[0].Status.Allocatable.DeepCopy(), // assuming all nodes have the same Allocatable ResourceList } - return r.client.Create(context.TODO(), dnp) + return r.client.Create(context.Background(), dnp) } func (r *reconciler) updateNodePool(onp *v1alpha1.NodePool, nodes []corev1.Node) error { // only update NodeAllocatable (price and min/max node counts to be updated by user, for now) - if reflect.DeepEqual(onp.Spec.NodeAllocatable, nodes[0].Status.Allocatable) { + if len(nodes) == 0 || reflect.DeepEqual(onp.Spec.NodeAllocatable, nodes[0].Status.Allocatable) { return nil } onp.Spec.NodeAllocatable = nodes[0].Status.Allocatable.DeepCopy() // assuming all selected nodes have the same Allocatable ResourceList - return r.client.Update(context.TODO(), onp) + return r.client.Update(context.Background(), onp) } diff --git a/pkg/controllers/receive/handler.go b/pkg/controllers/receive/handler.go deleted file mode 100644 index 1422f7b9..00000000 --- a/pkg/controllers/receive/handler.go +++ /dev/null @@ -1,58 +0,0 @@ -/* -Copyright 2019 The Multicluster-Scheduler Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package receive - -import ( - "admiralty.io/multicluster-controller/pkg/handler" - "admiralty.io/multicluster-controller/pkg/reconcile" - "admiralty.io/multicluster-controller/pkg/reference" - "k8s.io/apimachinery/pkg/api/meta" -) - -var key string = "multicluster.admiralty.io/controller-reference" - -type EnqueueRequestForMulticlusterController struct { - Queue handler.Queue -} - -func (e *EnqueueRequestForMulticlusterController) enqueue(obj interface{}) { - o, err := meta.Accessor(obj) - if err != nil { - return - } - - if c := reference.GetMulticlusterControllerOf(o); c != nil { - r := reconcile.Request{Context: c.ClusterName} - r.Namespace = c.Namespace - r.Name = c.Name - - e.Queue.Add(r) - return - } -} - -func (e *EnqueueRequestForMulticlusterController) OnAdd(obj interface{}) { - e.enqueue(obj) -} - -func (e *EnqueueRequestForMulticlusterController) OnUpdate(oldObj, newObj interface{}) { - e.enqueue(newObj) -} - -func (e *EnqueueRequestForMulticlusterController) OnDelete(obj interface{}) { - e.enqueue(obj) -} diff --git a/pkg/controllers/receive/receive.go b/pkg/controllers/receive/receive.go index 7cc4e69a..c592c65e 100644 --- a/pkg/controllers/receive/receive.go +++ b/pkg/controllers/receive/receive.go @@ -17,190 +17,74 @@ limitations under the License. package receive import ( - "context" - "fmt" - "admiralty.io/multicluster-controller/pkg/cluster" "admiralty.io/multicluster-controller/pkg/controller" - "admiralty.io/multicluster-controller/pkg/reconcile" - "admiralty.io/multicluster-controller/pkg/reference" - "admiralty.io/multicluster-scheduler/pkg/apis" - "k8s.io/apimachinery/pkg/api/errors" + "admiralty.io/multicluster-controller/pkg/patterns/gc" + "admiralty.io/multicluster-scheduler/pkg/apis/multicluster/v1alpha1" + "admiralty.io/multicluster-scheduler/pkg/common" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" ) -func NewController(agent *cluster.Cluster, scheduler *cluster.Cluster, decisionType runtime.Object, delegateType runtime.Object) (*controller.Controller, error) { - agentClient, err := agent.GetDelegatingClient() - if err != nil { - return nil, fmt.Errorf("getting delegating client for agent cluster: %v", err) - } - schedulerClient, err := scheduler.GetDelegatingClient() - if err != nil { - return nil, fmt.Errorf("getting delegating client for scheduler cluster: %v", err) - } - - decisionGVKs, _, err := scheduler.GetScheme().ObjectKinds(decisionType) - if err != nil { - return nil, fmt.Errorf("getting decision group version kind: %v", err) - } - delegateGVKs, _, err := scheduler.GetScheme().ObjectKinds(delegateType) - if err != nil { - return nil, fmt.Errorf("getting delegate group version kind: %v", err) - } - - co := controller.New(&reconciler{ - agent: agentClient, - scheduler: schedulerClient, - agentContext: agent.Name, - decisionGVK: decisionGVKs[0], // TODO... get preferred GVK if many - delegateGVK: delegateGVKs[0], // TODO... get preferred GVK if many - }, controller.Options{}) - - if err := apis.AddToScheme(scheduler.GetScheme()); err != nil { - return nil, fmt.Errorf("adding APIs to scheduler cluster's scheme: %v", err) - } - if err := co.WatchResourceReconcileObject(scheduler, decisionType, controller.WatchOptions{}); err != nil { - return nil, fmt.Errorf("setting up decision watch on scheduler cluster: %v", err) - } - - if err := apis.AddToScheme(agent.GetScheme()); err != nil { - return nil, fmt.Errorf("adding APIs to agent cluster's scheme: %v", err) - } - // if err := co.WatchResourceReconcileController(agent, delegateType, controller.WatchOptions{}); err != nil { - // return nil, fmt.Errorf("setting up delegate watch on agent cluster: %v", err) - // } - // TODO: when multicluster-controller implements it, use WatchResourceReconcileMulticlusterController - h := &EnqueueRequestForMulticlusterController{Queue: co.Queue} - if err := agent.AddEventHandler(delegateType, h); err != nil { - return nil, err - } - - return co, nil +var AllDecisions = map[runtime.Object]runtime.Object{ + &v1alpha1.PodDecision{}: &corev1.Pod{}, + &v1alpha1.ServiceDecision{}: &corev1.Service{}, } -type reconciler struct { - agent client.Client - scheduler client.Client - agentContext string - decisionGVK schema.GroupVersionKind - delegateGVK schema.GroupVersionKind +func NewController(agent *cluster.Cluster, scheduler *cluster.Cluster, decisionNamespace string, + decisionType runtime.Object, delegateType runtime.Object) (*controller.Controller, error) { + return gc.NewController(scheduler, agent, gc.Options{ + ParentPrototype: decisionType, + ChildPrototype: delegateType, + ParentWatchOptions: controller.WatchOptions{ + Namespace: decisionNamespace, + AnnotationSelector: labels.SelectorFromValidatedSet(labels.Set{ + common.AnnotationKeyClusterName: agent.GetClusterName()})}, + Applier: applier{}, + }) } -func (r *reconciler) Reconcile(req reconcile.Request) (reconcile.Result, error) { - decName := req.Name - decNamespace := req.Namespace +type applier struct{} - dec := &unstructured.Unstructured{} - dec.SetGroupVersionKind(r.decisionGVK) - if err := r.scheduler.Get(context.TODO(), req.NamespacedName, dec); err != nil { - if !errors.IsNotFound(err) { - return reconcile.Result{}, fmt.Errorf("cannot get decision %s in namespace %s in scheduler cluster: %v", decName, decNamespace, err) - } - // TODO? reconcile on child and delete child if parent doesn't exist anymore, - // in case we allow force deletes of parents in intermittent cluster connectivity use case. - return reconcile.Result{}, nil +var _ gc.Applier = applier{} + +func (a applier) MakeChild(parent interface{}, expectedChild interface{}) error { + dec, err := runtime.DefaultUnstructuredConverter.ToUnstructured(parent) + if err != nil { + return err // TODO + } + del, err := runtime.DefaultUnstructuredConverter.ToUnstructured(expectedChild) + if err != nil { + return err // TODO } - tmplMeta, found, err := unstructured.NestedMap(dec.Object, "spec", "template", "metadata") + meta, found, err := unstructured.NestedMap(dec, "spec", "template", "metadata") if err != nil || !found { panic("bad format") // as in impossible } - clusterName, found, err := unstructured.NestedString(tmplMeta, "clusterName") - if err != nil { + spec, found, err := unstructured.NestedFieldCopy(dec, "spec", "template", "spec") + if err != nil || !found { panic("bad format") // as in impossible } - if !found { - return reconcile.Result{}, fmt.Errorf("decision %s in namespace %s template missing cluster name: %v", decName, decNamespace, err) - } - if clusterName != r.agentContext { - // request for other cluster, do nothing - // TODO: filter upstream (with Watch predicate) - return reconcile.Result{}, nil - } - delName, found, err := unstructured.NestedString(tmplMeta, "name") - if err != nil { + if err := unstructured.SetNestedField(del, meta, "metadata"); err != nil { panic("bad format") // as in impossible } - if !found { - return reconcile.Result{}, fmt.Errorf("decision %s in namespace %s template missing name: %v", decName, decNamespace, err) - } - delNamespace, found, err := unstructured.NestedString(tmplMeta, "namespace") - if err != nil { + if err := unstructured.SetNestedField(del, spec, "spec"); err != nil { panic("bad format") // as in impossible } - if !found { - return reconcile.Result{}, fmt.Errorf("decision %s in namespace %s template missing namespace: %v", decName, decNamespace, err) - } - delFound := true - del := &unstructured.Unstructured{} - del.SetGroupVersionKind(r.delegateGVK) - if err := r.agent.Get(context.TODO(), types.NamespacedName{Name: delName, Namespace: delNamespace}, del); err != nil { - if !errors.IsNotFound(err) { - return reconcile.Result{}, fmt.Errorf("cannot get delegate %s in namespace %s in agent cluster: %v", delName, delNamespace, err) - } - delFound = false - } - - decTerminating := dec.GetDeletionTimestamp() != nil - - finalizers := dec.GetFinalizers() - j := -1 - for i, f := range finalizers { - if f == "multiclusterForegroundDeletion" { - j = i - break - } - } - decHasFinalizer := j > -1 - - if decTerminating { - if delFound { - if err := r.agent.Delete(context.TODO(), del); err != nil && !errors.IsNotFound(err) { - return reconcile.Result{}, fmt.Errorf("cannot delete delegate %s in namespace %s in agent cluster: %v", delName, delNamespace, err) - } - } else if decHasFinalizer { - // remove finalizer - dec.SetFinalizers(append(finalizers[:j], finalizers[j+1:]...)) - if err := r.scheduler.Update(context.TODO(), dec); err != nil { - return reconcile.Result{}, fmt.Errorf("cannot remove finalizer from decision %s in namespace %s in scheduler cluster: %v", decName, decNamespace, err) - } - } - } else { - if !decHasFinalizer { - dec.SetFinalizers(append(finalizers, "multiclusterForegroundDeletion")) - if err := r.scheduler.Update(context.TODO(), dec); err != nil { - return reconcile.Result{}, fmt.Errorf("cannot add finalizer to decision %s in namespace %s in scheduler cluster: %v", decName, decNamespace, err) - } - } else if !delFound { - // create child only after multicluster GC finalizer has been set - del := &unstructured.Unstructured{} - del.SetGroupVersionKind(r.delegateGVK) - if err := unstructured.SetNestedField(del.Object, tmplMeta, "metadata"); err != nil { - panic("bad format") // as in impossible - } - spec, found, err := unstructured.NestedFieldCopy(dec.Object, "spec", "template", "spec") // TODO error - if err != nil || !found { - panic("bad format") // as in impossible - } - if err := unstructured.SetNestedField(del.Object, spec, "spec"); err != nil { - panic("bad format") // as in impossible - } - - ref := reference.NewMulticlusterOwnerReference(dec, dec.GroupVersionKind(), req.Context) - reference.SetMulticlusterControllerReference(del, ref) - - if err := r.agent.Create(context.TODO(), del); err != nil && !errors.IsAlreadyExists(err) { - return reconcile.Result{}, fmt.Errorf("cannot create delegate %s in namespace %s in agent cluster: %v", delName, delNamespace, err) - } - } + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(del, expectedChild); err != nil { + return err // TODO } + return nil +} - // TODO: smart delegate pod update (only the allowed fields: container and initContainer images, etc.) +func (a applier) ChildNeedsUpdate(interface{}, interface{}, interface{}) (bool, error) { + return false, nil +} - return reconcile.Result{}, nil +func (a applier) MutateChild(interface{}, interface{}, interface{}) error { + return nil } diff --git a/pkg/controllers/schedule/schedule.go b/pkg/controllers/schedule/schedule.go index d9ec4225..65e37692 100644 --- a/pkg/controllers/schedule/schedule.go +++ b/pkg/controllers/schedule/schedule.go @@ -17,280 +17,115 @@ limitations under the License. package schedule import ( - "context" "fmt" - "strings" "admiralty.io/multicluster-controller/pkg/cluster" "admiralty.io/multicluster-controller/pkg/controller" - "admiralty.io/multicluster-controller/pkg/reconcile" - "admiralty.io/multicluster-controller/pkg/reference" - "admiralty.io/multicluster-scheduler/pkg/apis" + "admiralty.io/multicluster-controller/pkg/patterns/decorator" "admiralty.io/multicluster-scheduler/pkg/apis/multicluster/v1alpha1" "admiralty.io/multicluster-scheduler/pkg/common" + schedulerconfig "admiralty.io/multicluster-scheduler/pkg/config/scheduler" "github.com/ghodss/yaml" - "github.com/go-test/deep" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "k8s.io/apimachinery/pkg/util/runtime" ) -func NewController(local *cluster.Cluster, s Scheduler) (*controller.Controller, error) { - client, err := local.GetDelegatingClient() +func NewController(c *cluster.Cluster, s Scheduler, schedCfg *schedulerconfig.Config) (*controller.Controller, error) { + pendingDecisions := make(pendingDecisions) + client, err := c.GetDelegatingClient() if err != nil { - return nil, fmt.Errorf("getting delegating client for local cluster: %v", err) - } - - co := controller.New(&reconciler{ - client: client, - scheme: local.GetScheme(), - scheduler: s, - pendingDecisions: make(map[string]*v1alpha1.PodDecision), - }, controller.Options{}) - - if err := apis.AddToScheme(local.GetScheme()); err != nil { - return nil, fmt.Errorf("adding APIs to local cluster's scheme: %v", err) - } - if err := co.WatchResourceReconcileObject(local, &v1alpha1.PodObservation{}, controller.WatchOptions{}); err != nil { - return nil, fmt.Errorf("setting up proxy pod observation watch: %v", err) - } // TODO: filter on annotation (proxy pod observations only) - if err := co.WatchResourceReconcileController(local, &v1alpha1.PodDecision{}, controller.WatchOptions{}); err != nil { - return nil, fmt.Errorf("setting up delegate pod decision watch: %v", err) - } - - return co, nil + return nil, fmt.Errorf("getting delegating client: %v", err) + } + return decorator.NewController(c, &v1alpha1.PodObservation{}, applier{ + schedCfg: schedCfg, + scheduler: schedulerShim{ + schedCfg: schedCfg, + client: client, + pendingDecisions: pendingDecisions, + scheduler: s, + }, + pendingDecisions: pendingDecisions, + }, controller.WatchOptions{Namespaces: schedCfg.Namespaces}) } -type reconciler struct { - client client.Client - scheme *runtime.Scheme - scheduler Scheduler - pendingDecisions map[string]*v1alpha1.PodDecision // Note: this makes the reconciler NOT compatible with MaxConccurentReconciles > 1 +type applier struct { + schedCfg *schedulerconfig.Config + scheduler SchedulerShim + pendingDecisions pendingDecisions + // Note: this makes the reconciler NOT compatible with MaxConccurentReconciles > 1 + // TODO add mutex if we want concurrent reconcilers } -func (r *reconciler) Reconcile(req reconcile.Request) (reconcile.Result, error) { - proxyPodObs := &v1alpha1.PodObservation{} - if err := r.client.Get(context.TODO(), req.NamespacedName, proxyPodObs); err != nil { - if !errors.IsNotFound(err) { - return reconcile.Result{}, fmt.Errorf("cannot get proxy pod observation %s in namespace %s: %v", req.Name, req.Namespace, err) - } - // PodObservation was deleted: - // PodDecision will be garbage-collected after the corresponding agent has deleted the delegate Pod - // and removed the finalizer from the PodDecision. - return reconcile.Result{}, nil - } - - proxyPod := proxyPodObs.Status.LiveState - if _, ok := proxyPod.Annotations[common.AnnotationKeyElect]; !ok { +func (r applier) NeedUpdate(obj interface{}) (bool, error) { + podObs := obj.(*v1alpha1.PodObservation) + pod := podObs.Status.LiveState + if _, ok := pod.Annotations[common.AnnotationKeyElect]; !ok { // not a proxy pod // but it could be a delegate pod, in which case we want to remove the corresponding pod decision from the pending map - ref := reference.GetMulticlusterControllerOf(proxyPod) - if ref != nil && ref.Kind == "PodDecision" { - delete(r.pendingDecisions, ref.Name) + k := key{ + clusterName: r.schedCfg.GetObservationClusterName(podObs), + namespace: pod.Namespace, + name: pod.Name, } + delete(r.pendingDecisions, k) - return reconcile.Result{}, nil - } - - delegatePod, err := r.makeDelegatePod(proxyPodObs) - if err != nil { - return reconcile.Result{}, fmt.Errorf("cannot make delegate pod from proxy pod observation %s in namespace %s: %v", req.Name, req.Namespace, err) - } - - delegatePodDec := &v1alpha1.PodDecision{} - if err := r.client.Get(context.TODO(), req.NamespacedName, delegatePodDec); err != nil { - if !errors.IsNotFound(err) { - return reconcile.Result{}, fmt.Errorf("cannot get delegate pod decision %s in namespace %s: %v", req.Name, req.Namespace, err) - } - return reconcile.Result{}, r.createDelegatePodDecision(delegatePod, proxyPodObs) + return false, nil } - // no rescheduling - delegatePod.ClusterName = delegatePodDec.Spec.Template.ClusterName - if needUpdate(delegatePodDec, delegatePod) { - return reconcile.Result{}, r.updateDelegatePodDecision(delegatePodDec, delegatePod) + clusterName := pod.Annotations[common.AnnotationKeyClusterName] + if clusterName != "" { + // already scheduled + // bind controller will check if allowed + return false, nil } - return reconcile.Result{}, nil + return true, nil } -func (r *reconciler) makeDelegatePod(proxyPodObs *v1alpha1.PodObservation) (*corev1.Pod, error) { - proxyPod := proxyPodObs.Status.LiveState - srcPodManifest, ok := proxyPod.Annotations[common.AnnotationKeySourcePodManifest] +func (r applier) Mutate(obj interface{}) error { + podObs := obj.(*v1alpha1.PodObservation) + pod := podObs.Status.LiveState + srcPodManifest, ok := pod.Annotations[common.AnnotationKeySourcePodManifest] if !ok { - return nil, fmt.Errorf("no source pod manifest on proxy pod") + return fmt.Errorf("no source pod manifest on proxy pod") } srcPod := &corev1.Pod{} if err := yaml.Unmarshal([]byte(srcPodManifest), srcPod); err != nil { - return nil, fmt.Errorf("cannot unmarshal source pod manifest: %v", err) + return fmt.Errorf("cannot unmarshal source pod manifest: %v", err) } - annotations := make(map[string]string) - for k, v := range srcPod.Annotations { - if k != common.AnnotationKeyElect { // we don't want to mc-schedule the delegate pod - annotations[k] = v - } - } - annotations[common.AnnotationKeyProxyPodClusterName] = proxyPod.ClusterName - annotations[common.AnnotationKeyProxyPodNamespace] = proxyPod.Namespace - annotations[common.AnnotationKeyProxyPodName] = proxyPod.Name - - labels := make(map[string]string) - for k, v := range srcPod.Labels { - // we need to change the labels so as not to confuse potential controller of proxy pod, e.g., replica set - // if the original label key has a domain prefix, replace it with ours - // if it doesn't, add our domain prefix - keySplit := strings.Split(k, "/") // note: assume no empty key (enforced by Kubernetes) - newKey := common.KeyPrefix + keySplit[len(keySplit)-1] - labels[newKey] = v - } - - delegatePod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: proxyPodObs.Name, - Namespace: proxyPod.Namespace, // already defaults to "default" (vs. could be empty in srcPod) - ClusterName: proxyPod.ClusterName, - Labels: labels, - Annotations: annotations}, - Spec: *srcPod.Spec.DeepCopy()} - - removeServiceAccount(delegatePod) - // TODO? add compatible fields instead of removing incompatible ones - // (for forward compatibility and we've certainly forgotten incompatible fields...) - // TODO... maybe make this configurable, sort of like Federation v2 Overrides - - r.scheduler.Reset() - if err := r.getObservations(); err != nil { - return nil, fmt.Errorf("cannot get observations: %v", err) - } - - if clusterName, ok := annotations[common.AnnotationKeyClusterName]; ok { - delegatePod.ClusterName = clusterName - } else { + srcClusterName := r.schedCfg.GetObservationClusterName(podObs) + srcPod.ClusterName = srcClusterName // so basic scheduler's Schedule() can default to source cluster + // if no cluster can accommodate the delegate pod, Schedule() sets clusterName to srcPod.ClusterName + clusterName, err := r.scheduler.Schedule(srcPod) + if err != nil { // TODO: pod observations pending cluster if not enough resources and node pools not elastic // rather than send back to original cluster - clusterName, err := r.scheduler.Schedule(delegatePod) - if err != nil { - return nil, fmt.Errorf("cannot schedule: %v", err) - } - delegatePod.ClusterName = clusterName + // basic scheduler's Schedule() handles error already, but a different implementation could return an error + runtime.HandleError(fmt.Errorf("cannot schedule proxy pod observation %s in namespace %s (handled: scheduling to original cluster instead): %v", podObs.Name, podObs.Namespace, err)) + clusterName = srcClusterName } - return delegatePod, nil -} + srcPod.ClusterName = clusterName // set ClusterName because scheduler expects it on pending decisions + r.pendingDecisions[key{ + clusterName: clusterName, + namespace: pod.Namespace, + name: pod.Name, + }] = srcPod -func removeServiceAccount(pod *corev1.Pod) { - var saSecretName string - for i, c := range pod.Spec.Containers { - j := -1 - for i, m := range c.VolumeMounts { - if m.MountPath == "/var/run/secrets/kubernetes.io/serviceaccount" { - saSecretName = m.Name - j = i - break - } - } - if j > -1 { - c.VolumeMounts = append(c.VolumeMounts[:j], c.VolumeMounts[j+1:]...) - pod.Spec.Containers[i] = c - } - } - j := -1 - for i, v := range pod.Spec.Volumes { - if v.Name == saSecretName { - j = i - break - } - } - if j > -1 { - pod.Spec.Volumes = append(pod.Spec.Volumes[:j], pod.Spec.Volumes[j+1:]...) - } -} - -func (r *reconciler) getObservations() error { - podObsL := &v1alpha1.PodObservationList{} - if err := r.client.List(context.TODO(), &client.ListOptions{}, podObsL); err != nil { - return fmt.Errorf("cannot list pod observations: %v", err) - } - for _, podObs := range podObsL.Items { - phase := podObs.Status.LiveState.Status.Phase - if phase == corev1.PodPending || phase == corev1.PodRunning { - r.scheduler.SetPod(podObs.Status.LiveState) - } - } - - // get pending pod decisions so we can count the requests (cluster targeted but no pod obs yet) - // otherwise a bunch of pods would be scheduled to one cluster before it would appear to be busy - for _, podDec := range r.pendingDecisions { - r.scheduler.SetPod(&corev1.Pod{ - ObjectMeta: podDec.Spec.Template.ObjectMeta, - Spec: podDec.Spec.Template.Spec, - }) - } - - nodeObsL := &v1alpha1.NodeObservationList{} - if err := r.client.List(context.TODO(), &client.ListOptions{}, nodeObsL); err != nil { - return fmt.Errorf("cannot list node observations: %v", err) - } - for _, nodeObs := range nodeObsL.Items { - r.scheduler.SetNode(nodeObs.Status.LiveState) - } - - npObsL := &v1alpha1.NodePoolObservationList{} - if err := r.client.List(context.TODO(), &client.ListOptions{}, npObsL); err != nil { - return fmt.Errorf("cannot list node pool observations: %v", err) - } - for _, npObs := range npObsL.Items { - r.scheduler.SetNodePool(npObs.Status.LiveState) - } - - return nil -} - -func (r *reconciler) createDelegatePodDecision(delegatePod *corev1.Pod, proxyPodObs *v1alpha1.PodObservation) error { - delegatePodDec := &v1alpha1.PodDecision{} - delegatePodDec.Namespace = proxyPodObs.Namespace - delegatePodDec.Name = proxyPodObs.Name - delegatePodDec.Spec.Template.ObjectMeta = *delegatePod.ObjectMeta.DeepCopy() - delegatePodDec.Spec.Template.Spec = *delegatePod.Spec.DeepCopy() - if err := controllerutil.SetControllerReference(proxyPodObs, delegatePodDec, r.scheme); err != nil { - return fmt.Errorf("cannot set controller reference on delegate pod decision %s in namespace %s for owner %s in namespace %s: %v", - delegatePodDec.Name, delegatePodDec.Namespace, proxyPodObs.Name, proxyPodObs.Namespace, err) - } - if err := r.client.Create(context.TODO(), delegatePodDec); err != nil { - return fmt.Errorf("cannot create delegate pod decision %s in namespace %s: %v", delegatePodDec.Name, delegatePodDec.Namespace, err) - } - r.pendingDecisions[delegatePodDec.Name] = delegatePodDec + podObs.Status.LiveState.Annotations[common.AnnotationKeyClusterName] = clusterName return nil } -func needUpdate(delegatePodDec *v1alpha1.PodDecision, delegatePod *corev1.Pod) bool { - if diff := deep.Equal(delegatePodDec.Spec.Template.ObjectMeta, delegatePod.ObjectMeta); diff != nil { - return true - } - if diff := deep.Equal(delegatePodDec.Spec.Template.Spec, delegatePod.Spec); diff != nil { - return true - } - return false -} +type pendingDecisions map[key]*corev1.Pod -func (r *reconciler) updateDelegatePodDecision(delegatePodDec *v1alpha1.PodDecision, delegatePod *corev1.Pod) error { - delegatePodDec.Spec.Template.ObjectMeta = *delegatePod.ObjectMeta.DeepCopy() - delegatePodDec.Spec.Template.Spec = *delegatePod.Spec.DeepCopy() - if err := r.client.Update(context.TODO(), delegatePodDec); err != nil { - return fmt.Errorf("cannot update delegate pod decision %s in namespace %s: %v", delegatePodDec.Name, delegatePodDec.Namespace, err) - } - return nil +type key struct { + clusterName string + namespace string + name string } -type Scheduler interface { - Reset() - SetPod(p *corev1.Pod) - SetNode(n *corev1.Node) - SetNodePool(np *v1alpha1.NodePool) - Schedule(p *corev1.Pod) (string, error) +type SchedulerShim interface { + Schedule(pod *corev1.Pod) (string, error) } diff --git a/pkg/controllers/schedule/schedule_test.go b/pkg/controllers/schedule/schedule_test.go new file mode 100644 index 00000000..84513de9 --- /dev/null +++ b/pkg/controllers/schedule/schedule_test.go @@ -0,0 +1,239 @@ +package schedule + +import ( + "fmt" + "testing" + + configv1alpha1 "admiralty.io/multicluster-scheduler/pkg/apis/config/v1alpha1" + "admiralty.io/multicluster-scheduler/pkg/apis/multicluster/v1alpha1" + "admiralty.io/multicluster-scheduler/pkg/common" + "admiralty.io/multicluster-scheduler/pkg/config/scheduler" + "github.com/ghodss/yaml" + "github.com/go-test/deep" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var schedulerNamespace = "multicluster-scheduler" + +var testCases = map[string]struct { + applier applier + podObsBefore *v1alpha1.PodObservation + srcPod *corev1.Pod // serialization added as annotation before test + podObsAfter *v1alpha1.PodObservation // nil if no update + pendingDecisionsAfter pendingDecisions +}{ + "normal": { + applier{ + schedCfg: scheduler.New(&configv1alpha1.Scheduler{ + UseClusterNamespaces: true, + Clusters: []configv1alpha1.Cluster{ + {Name: "c1"}, + {Name: "c2"}, + }, + }, schedulerNamespace), + scheduler: testScheduler{"c2", nil}, + pendingDecisions: pendingDecisions{}, + }, + &v1alpha1.PodObservation{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "c1", + }, + Status: v1alpha1.PodObservationStatus{ + LiveState: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "ns1", + Annotations: map[string]string{common.AnnotationKeyElect: ""}, + }, + }, + }, + }, + &corev1.Pod{}, + &v1alpha1.PodObservation{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "c1", + }, + Status: v1alpha1.PodObservationStatus{ + LiveState: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "ns1", + Annotations: map[string]string{ + common.AnnotationKeyElect: "", + common.AnnotationKeyClusterName: "c2", + }, + }, + }, + }, + }, + pendingDecisions{ + key{"c2", "ns1", "pod1"}: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ClusterName: "c2"}, + }, + }, + }, + "remove from pending decisions": { + applier{ + schedCfg: scheduler.New(&configv1alpha1.Scheduler{ + UseClusterNamespaces: true, + Clusters: []configv1alpha1.Cluster{ + {Name: "c1"}, + {Name: "c2"}, + }, + }, schedulerNamespace), + pendingDecisions: pendingDecisions{ + key{"c2", "ns1", "pod1"}: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ClusterName: "c2"}, + }, + }, + }, + &v1alpha1.PodObservation{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "c2", + }, + Status: v1alpha1.PodObservationStatus{ + LiveState: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "ns1", + }, + }, + }, + }, + nil, + nil, + pendingDecisions{}, + }, + "already scheduled": { + applier{ + schedCfg: scheduler.New(&configv1alpha1.Scheduler{ + UseClusterNamespaces: true, + Clusters: []configv1alpha1.Cluster{ + {Name: "c1"}, + {Name: "c2"}, + }, + }, schedulerNamespace), + pendingDecisions: pendingDecisions{}, + }, + &v1alpha1.PodObservation{ + Status: v1alpha1.PodObservationStatus{ + LiveState: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + common.AnnotationKeyElect: "", + common.AnnotationKeyFederationName: "default", + common.AnnotationKeyClusterName: "c3", + }, + }, + }, + }, + }, + nil, + nil, + pendingDecisions{}, + }, + "return to original cluster on scheduling error": { + applier{ + schedCfg: scheduler.New(&configv1alpha1.Scheduler{ + UseClusterNamespaces: true, + Clusters: []configv1alpha1.Cluster{ + {Name: "c1"}, + {Name: "c2"}, + }, + }, schedulerNamespace), + scheduler: testScheduler{"", fmt.Errorf("some error")}, + pendingDecisions: pendingDecisions{}, + }, + &v1alpha1.PodObservation{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "c1", + }, + Status: v1alpha1.PodObservationStatus{ + LiveState: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "ns1", + Annotations: map[string]string{ + common.AnnotationKeyElect: "", + }, + }, + }, + }, + }, + &corev1.Pod{}, + &v1alpha1.PodObservation{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "c1", + }, + Status: v1alpha1.PodObservationStatus{ + LiveState: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "ns1", + Annotations: map[string]string{ + common.AnnotationKeyElect: "", + common.AnnotationKeyClusterName: "c1", + }, + }, + }, + }, + }, + pendingDecisions{ + key{"c1", "ns1", "pod1"}: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ClusterName: "c1"}, + }, + }, + }, +} + +type testScheduler struct { + clusterName string + err error +} + +var _ SchedulerShim = testScheduler{} + +func (s testScheduler) Schedule(*corev1.Pod) (string, error) { + return s.clusterName, s.err +} + +func TestSchedule(t *testing.T) { + for k, v := range testCases { + if v.podObsAfter != nil { + srcPodManifest, err := yaml.Marshal(v.srcPod) + if err != nil { + t.Errorf("%s failed: cannot serialize source pod: %v", k, err) + } + v.podObsBefore.Status.LiveState.Annotations[common.AnnotationKeySourcePodManifest] = string(srcPodManifest) + v.podObsAfter.Status.LiveState.Annotations[common.AnnotationKeySourcePodManifest] = string(srcPodManifest) + } + needUpdate, err := v.applier.NeedUpdate(v.podObsBefore) + if err != nil { + t.Errorf("%s failed: %v", k, err) + } + if needUpdate != (v.podObsAfter != nil) { + var shouldOrShouldNot string + if needUpdate { + shouldOrShouldNot = "should" + } else { + shouldOrShouldNot = "should not" + } + t.Errorf("%s failed: %s need update", k, shouldOrShouldNot) + } + if needUpdate { + err := v.applier.Mutate(v.podObsBefore) + if err != nil { + t.Errorf("%s failed: %v", k, err) + } + diff := deep.Equal(v.podObsBefore, v.podObsAfter) + if len(diff) > 0 { + t.Errorf("%s failed with pod obs diff: %v", k, diff) + } + } + diff := deep.Equal(v.pendingDecisionsAfter, v.applier.pendingDecisions) + if len(diff) > 0 { + t.Errorf("%s failed with pending decisions diff: %v", k, diff) + } + } +} diff --git a/pkg/controllers/schedule/scheduler_shim.go b/pkg/controllers/schedule/scheduler_shim.go new file mode 100644 index 00000000..4f350f9b --- /dev/null +++ b/pkg/controllers/schedule/scheduler_shim.go @@ -0,0 +1,100 @@ +package schedule + +import ( + "context" + "fmt" + + "admiralty.io/multicluster-scheduler/pkg/apis/multicluster/v1alpha1" + "admiralty.io/multicluster-scheduler/pkg/common" + schedulerconfig "admiralty.io/multicluster-scheduler/pkg/config/scheduler" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type schedulerShim struct { + client client.Client + pendingDecisions pendingDecisions + scheduler Scheduler + schedCfg *schedulerconfig.Config +} + +func (r schedulerShim) Schedule(srcPod *corev1.Pod) (string, error) { + r.scheduler.Reset() + + var namespaces []string + var clusters map[string]struct{} + fedName, _ := srcPod.Annotations[common.AnnotationKeyFederationName] + if fedName == "" { + srcClusterName := srcPod.ClusterName // set by applier depending on useClusterNamespaces + namespaces = r.schedCfg.PairedNamespacesByCluster[srcClusterName] + clusters = r.schedCfg.PairedClustersByCluster[srcClusterName] + } else { + namespaces = r.schedCfg.NamespacesByFederation[fedName] + clusters = r.schedCfg.ClustersByFederation[fedName] + } + for _, ns := range namespaces { + if err := r.getObservations(ns, clusters); err != nil { + return "", fmt.Errorf("cannot get observations: %v", err) + } + } + // get pending pod decisions so we can count the requests (cluster targeted but no pod obs yet) + // otherwise a bunch of pods would be scheduled to one cluster before it would appear to be busy + for _, pod := range r.pendingDecisions { + if _, ok := clusters[pod.ClusterName]; ok { + r.scheduler.SetPod(pod) + } + } + + return r.scheduler.Schedule(srcPod) +} + +func (r *schedulerShim) getObservations(namespace string, clusters map[string]struct{}) error { + podObsL := &v1alpha1.PodObservationList{} + if err := r.client.List(context.Background(), &client.ListOptions{Namespace: namespace}, podObsL); err != nil { + return fmt.Errorf("cannot list pod observations: %v", err) + } + for _, podObs := range podObsL.Items { + phase := podObs.Status.LiveState.Status.Phase + if phase == corev1.PodPending || phase == corev1.PodRunning { + pod := podObs.Status.LiveState + pod.ClusterName = r.schedCfg.GetObservationClusterName(&podObs) + if _, ok := clusters[pod.ClusterName]; ok { + r.scheduler.SetPod(pod) + } + } + } + + nodeObsL := &v1alpha1.NodeObservationList{} + if err := r.client.List(context.Background(), &client.ListOptions{Namespace: namespace}, nodeObsL); err != nil { + return fmt.Errorf("cannot list node observations: %v", err) + } + for _, nodeObs := range nodeObsL.Items { + node := nodeObs.Status.LiveState + node.ClusterName = r.schedCfg.GetObservationClusterName(&nodeObs) + if _, ok := clusters[node.ClusterName]; ok { + r.scheduler.SetNode(node) + } + } + + npObsL := &v1alpha1.NodePoolObservationList{} + if err := r.client.List(context.Background(), &client.ListOptions{Namespace: namespace}, npObsL); err != nil { + return fmt.Errorf("cannot list node pool observations: %v", err) + } + for _, npObs := range npObsL.Items { + np := npObs.Status.LiveState + np.ClusterName = r.schedCfg.GetObservationClusterName(&npObs) + if _, ok := clusters[np.ClusterName]; ok { + r.scheduler.SetNodePool(np) + } + } + + return nil +} + +type Scheduler interface { + Reset() + SetPod(p *corev1.Pod) + SetNode(n *corev1.Node) + SetNodePool(np *v1alpha1.NodePool) + Schedule(p *corev1.Pod) (string, error) +} diff --git a/pkg/controllers/send/send.go b/pkg/controllers/send/send.go index 3f66c96d..2772c429 100644 --- a/pkg/controllers/send/send.go +++ b/pkg/controllers/send/send.go @@ -17,253 +17,175 @@ limitations under the License. package send import ( - "context" "fmt" "reflect" "admiralty.io/multicluster-controller/pkg/cluster" "admiralty.io/multicluster-controller/pkg/controller" - "admiralty.io/multicluster-controller/pkg/reconcile" - "admiralty.io/multicluster-controller/pkg/reference" - "admiralty.io/multicluster-scheduler/pkg/apis" + "admiralty.io/multicluster-controller/pkg/patterns/gc" "admiralty.io/multicluster-scheduler/pkg/apis/multicluster/v1alpha1" "admiralty.io/multicluster-scheduler/pkg/common" - "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + policyv1beta1 "k8s.io/api/policy/v1beta1" + storagev1 "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" + "k8s.io/apimachinery/pkg/runtime/schema" ) -func NewController(agent *cluster.Cluster, scheduler *cluster.Cluster, federationNamespace string, +var AllObservations map[runtime.Object]runtime.Object + +func init() { + pvobs := &unstructured.Unstructured{} + pvobs.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "multicluster.admiralty.io", + Version: "v1alpha1", + Kind: "PersistentVolumeObservation"}) + pvcobs := &unstructured.Unstructured{} + pvcobs.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "multicluster.admiralty.io", + Version: "v1alpha1", + Kind: "PersistentVolumeClaimObservation"}) + rcobs := &unstructured.Unstructured{} + rcobs.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "multicluster.admiralty.io", + Version: "v1alpha1", + Kind: "ReplicationControllerObservation"}) + rsobs := &unstructured.Unstructured{} + rsobs.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "multicluster.admiralty.io", + Version: "v1alpha1", + Kind: "ReplicaSetObservation"}) + ssobs := &unstructured.Unstructured{} + ssobs.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "multicluster.admiralty.io", + Version: "v1alpha1", + Kind: "StatefulSetObservation"}) + pdbobs := &unstructured.Unstructured{} + pdbobs.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "multicluster.admiralty.io", + Version: "v1alpha1", + Kind: "PodDisruptionBudgetObservation"}) + scobs := &unstructured.Unstructured{} + scobs.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "multicluster.admiralty.io", + Version: "v1alpha1", + Kind: "StorageClassObservation"}) + + AllObservations = map[runtime.Object]runtime.Object{ + &corev1.Service{}: &v1alpha1.ServiceObservation{}, + &corev1.Pod{}: &v1alpha1.PodObservation{}, + &corev1.Node{}: &v1alpha1.NodeObservation{}, + &v1alpha1.NodePool{}: &v1alpha1.NodePoolObservation{}, + &corev1.PersistentVolume{}: pvobs, + &corev1.PersistentVolumeClaim{}: pvcobs, + &corev1.ReplicationController{}: rcobs, + &appsv1.ReplicaSet{}: rsobs, + &appsv1.StatefulSet{}: ssobs, + &policyv1beta1.PodDisruptionBudget{}: pdbobs, + &storagev1.StorageClass{}: scobs, + } +} + +func NewController(agent *cluster.Cluster, scheduler *cluster.Cluster, observationNamespace string, liveType runtime.Object, observationType runtime.Object) (*controller.Controller, error) { - agentclient, err := agent.GetDelegatingClient() + return gc.NewController(agent, scheduler, gc.Options{ + ParentPrototype: liveType, + ChildPrototype: observationType, + ChildNamespace: observationNamespace, + Applier: applier{}, + }) +} + +type applier struct{} + +var _ gc.Applier = applier{} + +func (a applier) MakeChild(parent interface{}, expectedChild interface{}) error { + live, err := runtime.DefaultUnstructuredConverter.ToUnstructured(parent) if err != nil { - return nil, fmt.Errorf("getting delegating client for agent cluster: %v", err) + return err // TODO } - schedulerclient, err := scheduler.GetDelegatingClient() + obs, err := runtime.DefaultUnstructuredConverter.ToUnstructured(expectedChild) if err != nil { - return nil, fmt.Errorf("getting delegating client for scheduler cluster: %v", err) + return err // TODO } - co := controller.New(&reconciler{ - agentContext: agent.Name, - agent: agentclient, - scheduler: schedulerclient, - federationNamespace: federationNamespace, - liveType: liveType, - observationType: observationType, - }, controller.Options{}) - - if err := apis.AddToScheme(agent.GetScheme()); err != nil { - return nil, fmt.Errorf("adding APIs to agent cluster's scheme: %v", err) - } - if err := co.WatchResourceReconcileObject(agent, liveType, controller.WatchOptions{}); err != nil { - return nil, fmt.Errorf("setting up live %T watch in agent cluster: %v", liveType, err) + if err := unstructured.SetNestedField(obs, map[string]interface{}{"liveState": live}, "status"); err != nil { + return fmt.Errorf("cannot set live state: %v", err) } - if err := apis.AddToScheme(scheduler.GetScheme()); err != nil { - return nil, fmt.Errorf("adding APIs to scheduler cluster's scheme: %v", err) + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obs, expectedChild); err != nil { + return err // TODO } - if err := co.WatchResourceReconcileController(scheduler, observationType, controller.WatchOptions{}); err != nil { - return nil, fmt.Errorf("setting up %T watch in scheduler cluster: %v", observationType, err) - } - - return co, nil + return nil } -type reconciler struct { - agentContext string - agent client.Client - scheduler client.Client - federationNamespace string - liveType runtime.Object - observationType runtime.Object -} +func (a applier) ChildNeedsUpdate(parent interface{}, child interface{}, _ interface{}) (bool, error) { + live := parent.(runtime.Object) + liveCopy := live.DeepCopyObject() + obs := child.(runtime.Object) + obsCopy := obs.DeepCopyObject() -func (r *reconciler) Reconcile(req reconcile.Request) (reconcile.Result, error) { - if req.Context != r.agentContext { - // request for other cluster, do nothing - // TODO: filter upstream (with Watch predicate) - return reconcile.Result{}, nil + liveCopyU, err := runtime.DefaultUnstructuredConverter.ToUnstructured(liveCopy) + if err != nil { + return false, err // TODO } - - obsNamespacedName := r.federationNamespacedName(req) - - live := r.liveType.DeepCopyObject() - if err := r.agent.Get(context.TODO(), req.NamespacedName, live); err != nil { - if !errors.IsNotFound(err) { - return reconcile.Result{}, fmt.Errorf("cannot get live %T %s in namespace %s in agent cluster: %v", - live, req.Name, req.Namespace, err) - } - // TODO...: multicluster garbage collector - // Until then... - return reconcile.Result{}, r.deleteObservation(obsNamespacedName) + obsCopyU, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obsCopy) + if err != nil { + return false, err // TODO } - setClusterName(live, req.Context) - obs := r.observationType.DeepCopyObject() - if err := r.scheduler.Get(context.TODO(), obsNamespacedName, obs); err != nil { - if !errors.IsNotFound(err) { - return reconcile.Result{}, fmt.Errorf("cannot get %T %s in namespace %s in scheduler cluster: %v", - obs, obsNamespacedName.Name, obsNamespacedName.Namespace, err) - } - obs, err := r.makeObservation(live, obsNamespacedName) - if err != nil { - return reconcile.Result{}, fmt.Errorf("cannot make observation from live %T %s in namespace %s in agent cluster: %v", - live, req.Name, req.Namespace, err) - } - if err := r.scheduler.Create(context.TODO(), obs); err != nil { - return reconcile.Result{}, fmt.Errorf("cannot create %T %s in namespace %s in scheduler cluster: %v", - obs, obsNamespacedName.Name, obsNamespacedName.Namespace, err) - } - return reconcile.Result{}, nil + obsLiveState, found, err := unstructured.NestedMap(obsCopyU, "status", "liveState") + if !found { + return false, err } - - ok, err := liveStateEqual(obs, live) if err != nil { - return reconcile.Result{}, fmt.Errorf("cannot compare %T to live %T: %v", obs, live, err) + return false, err } - if !ok { - if err := setLiveState(obs, live); err != nil { - return reconcile.Result{}, fmt.Errorf("cannot set live state of %T from %T: %v", obs, live, err) - } - if err := r.scheduler.Update(context.TODO(), obs); err != nil { - return reconcile.Result{}, fmt.Errorf("cannot update %T %s in namespace %s in scheduler cluster: %v", - obs, obsNamespacedName.Name, obsNamespacedName.Namespace, err) - } - return reconcile.Result{}, nil - } - - return reconcile.Result{}, nil -} -func (r *reconciler) federationNamespacedName(req reconcile.Request) types.NamespacedName { - name := req.Context - if req.Namespace != "" { - name += fmt.Sprintf("-%s", req.Namespace) + // target clustername annotation is updated by the scheduler, so don't use it for comparison + err = unstructured.SetNestedField(obsLiveState, "", "metadata", "annotations", common.AnnotationKeyClusterName) + if err != nil { + panic(err) } - name += fmt.Sprintf("-%s", req.Name) - - return types.NamespacedName{ - Namespace: r.federationNamespace, - Name: name, - // TODO: error if len(context) + len(namespace) + len(name) exceeds max. Name length - // or use GenerateName and Get observed controlled object by label or field selector (using List) + err = unstructured.SetNestedField(liveCopyU, "", "metadata", "annotations", common.AnnotationKeyClusterName) + if err != nil { + panic(err) } -} -// deleteObservation gets an observation before deleting. -// controller-runtime doesn't seem to offer the possibility to delete by namespaced name. -func (r *reconciler) deleteObservation(nn types.NamespacedName) error { - obs := r.observationType.DeepCopyObject() - if err := r.scheduler.Get(context.TODO(), nn, obs); err != nil { - if !errors.IsNotFound(err) { - return fmt.Errorf("cannot get (to delete) %T %s in namespace %s in scheduler cluster: %v", - obs, nn.Name, nn.Namespace, err) - } - // all good then - return nil - } - if err := r.scheduler.Delete(context.TODO(), obs); err != nil { - return fmt.Errorf("cannot delete %T %s in namespace %s in scheduler cluster: %v", - obs, nn.Name, nn.Namespace, err) - } - return nil + return !reflect.DeepEqual(liveCopyU, obsLiveState), nil } -func (r *reconciler) makeObservation(live runtime.Object, obsNamespacedName types.NamespacedName) (runtime.Object, error) { - var obs runtime.Object - switch live := live.(type) { - case *v1.Pod: - obs = &v1alpha1.PodObservation{} - case *v1.Node: - obs = &v1alpha1.NodeObservation{} - case *v1alpha1.NodePool: - obs = &v1alpha1.NodePoolObservation{} - case *v1.Service: - obs = &v1alpha1.ServiceObservation{} - default: - return nil, fmt.Errorf("type %T cannot be observed", live) +func (a applier) MutateChild(parent interface{}, child interface{}, _ interface{}) error { + liveU, err := runtime.DefaultUnstructuredConverter.ToUnstructured(parent) + if err != nil { + return err // TODO } - - if err := setLiveState(obs, live); err != nil { - return nil, fmt.Errorf("cannot set live state: %v", err) + obsU, err := runtime.DefaultUnstructuredConverter.ToUnstructured(child) + if err != nil { + return err // TODO } - liveMeta := live.(metav1.Object) - ref := reference.NewMulticlusterOwnerReference(liveMeta, live.GetObjectKind().GroupVersionKind(), r.agentContext) - obsMeta := obs.(metav1.Object) - reference.SetMulticlusterControllerReference(obsMeta, ref) - - obsMeta.SetNamespace(obsNamespacedName.Namespace) - obsMeta.SetName(obsNamespacedName.Name) - - labels := map[string]string{ - common.LabelKeyOriginalName: liveMeta.GetName(), - common.LabelKeyOriginalNamespace: liveMeta.GetNamespace(), - common.LabelKeyOriginalClusterName: liveMeta.GetClusterName(), + // backup target clustername annotation, which is updated by scheduler + targetClusterName, found, err := unstructured.NestedString(obsU, "status", "liveState", "metadata", "annotations", common.AnnotationKeyClusterName) + if err != nil { + panic(err) } - for k, v := range liveMeta.GetLabels() { - labels[k] = v + err = unstructured.SetNestedField(obsU, map[string]interface{}{"liveState": liveU}, "status") + if err != nil { + panic(err) } - obsMeta.SetLabels(labels) - - return obs, nil -} - -func setClusterName(live runtime.Object, clusterName string) { - meta := live.(metav1.Object) - meta.SetClusterName(clusterName) -} - -func liveStateEqual(obs runtime.Object, live runtime.Object) (bool, error) { - switch obs := obs.(type) { - case *v1alpha1.PodObservation: - return reflect.DeepEqual(live, obs.Status.LiveState), nil - case *v1alpha1.NodeObservation: - return reflect.DeepEqual(live, obs.Status.LiveState), nil - case *v1alpha1.NodePoolObservation: - return reflect.DeepEqual(live, obs.Status.LiveState), nil - case *v1alpha1.ServiceObservation: - return reflect.DeepEqual(live, obs.Status.LiveState), nil - default: - return false, fmt.Errorf("type %T is not an observation", obs) + if found { + err = unstructured.SetNestedField(obsU, targetClusterName, "status", "liveState", "metadata", "annotations", common.AnnotationKeyClusterName) + if err != nil { + panic(err) + } } -} -func setLiveState(obs runtime.Object, live runtime.Object) error { - switch obs := obs.(type) { - case *v1alpha1.PodObservation: - live, ok := live.(*v1.Pod) - if !ok { - return fmt.Errorf("type %T is not type %T's live form", live, obs) - } - obs.Status = v1alpha1.PodObservationStatus{LiveState: live} - return nil - case *v1alpha1.NodeObservation: - live, ok := live.(*v1.Node) - if !ok { - return fmt.Errorf("type %T is not type %T's live form", live, obs) - } - obs.Status = v1alpha1.NodeObservationStatus{LiveState: live} - return nil - case *v1alpha1.NodePoolObservation: - live, ok := live.(*v1alpha1.NodePool) - if !ok { - return fmt.Errorf("type %T is not type %T's live form", live, obs) - } - obs.Status = v1alpha1.NodePoolObservationStatus{LiveState: live} - return nil - case *v1alpha1.ServiceObservation: - live, ok := live.(*v1.Service) - if !ok { - return fmt.Errorf("type %T is not type %T's live form", live, obs) - } - obs.Status = v1alpha1.ServiceObservationStatus{LiveState: live} - return nil - default: - return fmt.Errorf("type %T is not an observation", obs) + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obsU, child); err != nil { + return err // TODO } + return nil } diff --git a/pkg/controllers/send/send_test.go b/pkg/controllers/send/send_test.go new file mode 100644 index 00000000..433c27cf --- /dev/null +++ b/pkg/controllers/send/send_test.go @@ -0,0 +1,208 @@ +package send + +import ( + "testing" + + "admiralty.io/multicluster-scheduler/pkg/apis/multicluster/v1alpha1" + "github.com/go-test/deep" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestMakeObservation(t *testing.T) { + testCases := map[string]struct { + live metav1.Object + obs metav1.Object + }{ + "normal": { + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + Namespace: "ns2", + Labels: map[string]string{"k3": "v3"}, + Annotations: map[string]string{"k4": "v4"}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + }, + }, + &v1alpha1.PodObservation{ + Status: v1alpha1.PodObservationStatus{ + LiveState: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + Namespace: "ns2", + Labels: map[string]string{"k3": "v3"}, + Annotations: map[string]string{"k4": "v4"}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + }, + }, + }, + }, + }, + } + + for k, v := range testCases { + a := applier{} + expectedObs := &v1alpha1.PodObservation{} + if err := a.MakeChild(v.live, expectedObs); err != nil { + t.Errorf("%s failed: %v", k, err) + } + diff := deep.Equal(v.obs, expectedObs) + if len(diff) > 0 { + t.Errorf("%s failed with observation diff: %v", k, diff) + } + } +} + +func TestObservationNeedsUpdate(t *testing.T) { + testCases := map[string]struct { + live metav1.Object + obs metav1.Object + needUpdate bool + }{ + "no change": { + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + Namespace: "ns2", + Labels: map[string]string{"k3": "v3"}, + Annotations: map[string]string{"k4": "v4"}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + }, + }, + &v1alpha1.PodObservation{ + Status: v1alpha1.PodObservationStatus{ + LiveState: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + Namespace: "ns2", + Labels: map[string]string{"k3": "v3"}, + Annotations: map[string]string{"k4": "v4"}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + }, + }, + }, + }, + false, + }, + "new annotation": { + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + Namespace: "ns2", + Labels: map[string]string{"k3": "v3"}, + Annotations: map[string]string{"k4": "v4", "k5": "v5"}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + }, + }, + &v1alpha1.PodObservation{ + Status: v1alpha1.PodObservationStatus{ + LiveState: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + Namespace: "ns2", + Labels: map[string]string{"k3": "v3"}, + Annotations: map[string]string{"k4": "v4"}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + }, + }, + }, + }, + true, + }, + "completed": { + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + Namespace: "ns2", + Labels: map[string]string{"k3": "v3"}, + Annotations: map[string]string{"k4": "v4"}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + }, + Status: corev1.PodStatus{ + Phase: "Succeeded", + }, + }, + &v1alpha1.PodObservation{ + Status: v1alpha1.PodObservationStatus{ + LiveState: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + Namespace: "ns2", + Labels: map[string]string{"k3": "v3"}, + Annotations: map[string]string{"k4": "v4"}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx", + }, + }, + }, + }, + }, + }, + true, + }, + } + + for k, v := range testCases { + a := applier{} + needUpdate, err := a.ChildNeedsUpdate(v.live, v.obs, &v1alpha1.PodObservation{}) + if err != nil { + t.Errorf("%s failed: %v", k, err) + } + if needUpdate != v.needUpdate { + t.Errorf("%s failed", k) + } + } +} diff --git a/pkg/controllers/svcreroute/svcreroute.go b/pkg/controllers/svcreroute/svcreroute.go index 9ea3b529..7227e513 100644 --- a/pkg/controllers/svcreroute/svcreroute.go +++ b/pkg/controllers/svcreroute/svcreroute.go @@ -23,6 +23,7 @@ import ( "admiralty.io/multicluster-controller/pkg/cluster" "admiralty.io/multicluster-controller/pkg/controller" + "admiralty.io/multicluster-controller/pkg/patterns" "admiralty.io/multicluster-controller/pkg/reconcile" "admiralty.io/multicluster-scheduler/pkg/common" corev1 "k8s.io/api/core/v1" @@ -41,9 +42,17 @@ func NewController(agent *cluster.Cluster) (*controller.Controller, error) { client: client, }, controller.Options{}) + // we watch endpoints to see if their listed pods are proxy pods if err := co.WatchResourceReconcileObject(agent, &corev1.Endpoints{}, controller.WatchOptions{}); err != nil { return nil, fmt.Errorf("setting up endpoints watch: %v", err) } + // we watch services because they are updated in the loop, + // and if those updates fail with an optimistic lock error + // we must requeue when we receive the cache is updated + if err := co.WatchResourceReconcileObject(agent, &corev1.Service{}, controller.WatchOptions{}); err != nil { + return nil, fmt.Errorf("setting up service watch: %v", err) + } + // a service and its endpoints object have the same name/namespace, i.e., the same reconcile key return co, nil } @@ -54,7 +63,7 @@ type reconciler struct { func (r *reconciler) Reconcile(req reconcile.Request) (reconcile.Result, error) { ep := &corev1.Endpoints{} - if err := r.client.Get(context.TODO(), req.NamespacedName, ep); err != nil { + if err := r.client.Get(context.Background(), req.NamespacedName, ep); err != nil { if !errors.IsNotFound(err) { return reconcile.Result{}, fmt.Errorf("cannot get endpoints %s in namespace %s: %v", req.Name, req.Namespace, err) } @@ -72,7 +81,7 @@ func (r *reconciler) Reconcile(req reconcile.Request) (reconcile.Result, error) } svc := &corev1.Service{} - if err := r.client.Get(context.TODO(), req.NamespacedName, svc); err != nil { + if err := r.client.Get(context.Background(), req.NamespacedName, svc); err != nil { if !errors.IsNotFound(err) { return reconcile.Result{}, fmt.Errorf("cannot get service %s in namespace %s: %v", req.Name, req.Namespace, err) } @@ -107,7 +116,7 @@ func (r *reconciler) Reconcile(req reconcile.Request) (reconcile.Result, error) return reconcile.Result{}, nil } - if err := r.client.Update(context.TODO(), svc); err != nil { + if err := r.client.Update(context.Background(), svc); err != nil && !patterns.IsOptimisticLockError(err) { return reconcile.Result{}, fmt.Errorf("cannot update service %s in namespace %s: %v", req.Name, req.Namespace, err) } @@ -120,7 +129,7 @@ func (r *reconciler) shouldReroute(ep *corev1.Endpoints) (bool, error) { if a.TargetRef != nil && a.TargetRef.Kind == "Pod" { key := types.NamespacedName{Name: a.TargetRef.Name, Namespace: a.TargetRef.Namespace} pod := &corev1.Pod{} - if err := r.client.Get(context.TODO(), key, pod); err != nil { + if err := r.client.Get(context.Background(), key, pod); err != nil { if !errors.IsNotFound(err) { return false, fmt.Errorf("cannot get pod %s in namespace %s: %v", key.Name, key.Namespace, err) } diff --git a/pkg/webhooks/proxypod/proxypod.go b/pkg/webhooks/proxypod/proxypod.go index 60a83b9b..37dab628 100644 --- a/pkg/webhooks/proxypod/proxypod.go +++ b/pkg/webhooks/proxypod/proxypod.go @@ -60,7 +60,10 @@ func NewServer(mgr manager.Manager, namespace string) (*webhook.Server, error) { Name: deployName, // Selectors should select the pods that runs this webhook server. Selectors: map[string]string{ - "app": deployName, + "app.kubernetes.io/name": deployName, + // HACK: there should be more here (cf. selectorLabels in helm chart) + // we could get the labels using the downward API, just like we get DEPLOYMENT_NAME + // but this won't be necessary once we stop using BootstrapOptions }, }, }, @@ -84,7 +87,7 @@ func NewWebhook(mgr manager.Manager) (*admission.Webhook, error) { Operations(admissionregistrationv1beta1.Create). // TODO: update (but careful not to proxy the proxy) WithManager(mgr). ForType(&corev1.Pod{}). - Handlers(&Handler{}). + Handlers(&Handler{mutator: mutator{}}). FailurePolicy(admissionregistrationv1beta1.Fail). NamespaceSelector(&metav1.LabelSelector{MatchLabels: map[string]string{"multicluster-scheduler": "enabled"}}). Build() @@ -93,6 +96,7 @@ func NewWebhook(mgr manager.Manager) (*admission.Webhook, error) { type Handler struct { decoder atypes.Decoder client client.Client + mutator mutator } func (h *Handler) Handle(ctx context.Context, req atypes.Request) atypes.Response { @@ -102,32 +106,46 @@ func (h *Handler) Handle(ctx context.Context, req atypes.Request) atypes.Respons return admission.ErrorResponse(http.StatusBadRequest, err) } - if _, ok := srcPod.Annotations[common.AnnotationKeyElect]; !ok { + proxyPod := srcPod.DeepCopy() + if err := h.mutator.mutate(proxyPod); err != nil { + return admission.ErrorResponse(http.StatusInternalServerError, err) + } + return admission.PatchResponse(srcPod, proxyPod) +} + +type mutator struct { +} + +func (m mutator) mutate(pod *corev1.Pod) error { + if _, ok := pod.Annotations[common.AnnotationKeyElect]; !ok { // not a multicluster pod - return admission.PatchResponse(srcPod, srcPod) + return nil } - srcPodManifest, err := yaml.Marshal(srcPod) + srcPodManifest, err := yaml.Marshal(pod) if err != nil { - return admission.ErrorResponse(http.StatusInternalServerError, err) + return err } - proxyPod := srcPod.DeepCopy() - // proxyPod.Annotations is not nil because we checked it contains AnnotationKeyElect - proxyPod.Annotations[common.AnnotationKeySourcePodManifest] = string(srcPodManifest) + // pod.Annotations is not nil because we checked it contains AnnotationKeyElect + pod.Annotations[common.AnnotationKeySourcePodManifest] = string(srcPodManifest) - for i, c := range proxyPod.Spec.Containers { // same number of containers because of jsonpatch bug - proxyPod.Spec.Containers[i] = corev1.Container{ + for i, c := range pod.Spec.Containers { + pod.Spec.Containers[i] = corev1.Container{ Name: c.Name, - Image: "busybox", - Command: []string{"sh", "-c", "trap 'exit 0' SIGUSR1; trap 'exit 1' SIGUSR2; (while sleep 3600; do :; done) & wait"}} + Image: image, + Command: command} // the feedback controller will send SIGUSR1 or SIGUSR2 when the delegate pod succeeds or fails, resp. } // TODO: add resource reqs/lims + other best practices + // TODO!!! remove scheduling prefs - return admission.PatchResponse(srcPod, proxyPod) + return nil } +var image = "busybox" +var command = []string{"sh", "-c", "trap 'exit 0' SIGUSR1; trap 'exit 1' SIGUSR2; (while sleep 3600; do :; done) & wait"} + // Handler implements inject.Client. // A client will be automatically injected. var _ inject.Client = &Handler{} diff --git a/pkg/webhooks/proxypod/proxypod_test.go b/pkg/webhooks/proxypod/proxypod_test.go new file mode 100644 index 00000000..34959733 --- /dev/null +++ b/pkg/webhooks/proxypod/proxypod_test.go @@ -0,0 +1,117 @@ +package proxypod + +import ( + "testing" + + "admiralty.io/multicluster-scheduler/pkg/common" + "github.com/ghodss/yaml" + "github.com/go-test/deep" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// TODO test webhook namespace selector + +var testCases = map[string]struct { + pod corev1.Pod + mutatedPod corev1.Pod +}{ + "proxy pod": { + corev1.Pod{ + ObjectMeta: v1.ObjectMeta{Annotations: map[string]string{common.AnnotationKeyElect: ""}}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: "nginx", + Image: "nginx", + }}}}, + corev1.Pod{ + ObjectMeta: v1.ObjectMeta{Annotations: map[string]string{ + common.AnnotationKeyElect: "", + common.AnnotationKeySourcePodManifest: "HACK", // yaml serialization computed in test code + }}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: "nginx", + Image: image, + Command: command, + }}}}, + }, + "other pod": { + corev1.Pod{ + Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: "nginx", + Image: "nginx", + }}}}, + corev1.Pod{ + Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: "nginx", + Image: "nginx", + }}}}, + }, + "federation name": { + corev1.Pod{ + ObjectMeta: v1.ObjectMeta{Annotations: map[string]string{ + common.AnnotationKeyElect: "", + common.AnnotationKeyFederationName: "f1", + }}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: "nginx", + Image: "nginx", + }}}}, + corev1.Pod{ + ObjectMeta: v1.ObjectMeta{Annotations: map[string]string{ + common.AnnotationKeyElect: "", + common.AnnotationKeyFederationName: "f1", + common.AnnotationKeySourcePodManifest: "HACK", // yaml serialization computed in test code + }}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: "nginx", + Image: image, + Command: command, + }}}}, + }, + "keep labels and annotations (in general, object meta)": { + corev1.Pod{ + ObjectMeta: v1.ObjectMeta{ + Annotations: map[string]string{common.AnnotationKeyElect: "", "k1": "v1"}, + Labels: map[string]string{"k2": "v2"}, + }, + Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: "nginx", + Image: "nginx", + }}}}, + corev1.Pod{ + ObjectMeta: v1.ObjectMeta{ + Annotations: map[string]string{ + common.AnnotationKeyElect: "", + common.AnnotationKeySourcePodManifest: "HACK", // yaml serialization computed in test code + "k1": "v1", + }, + Labels: map[string]string{"k2": "v2"}, + }, + Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: "nginx", + Image: image, + Command: command, + }}}}, + }, +} + +func TestMutate(t *testing.T) { + for k, v := range testCases { + podManifest, err := yaml.Marshal(v.pod) + if err != nil { + t.Errorf("%s failed: %v", k, err) + } + if k != "other pod" { + v.mutatedPod.Annotations[common.AnnotationKeySourcePodManifest] = string(podManifest) + } + m := mutator{} + mutatedPod := v.pod.DeepCopy() + if err := m.mutate(mutatedPod); err != nil { + t.Errorf("%s failed: %v", k, err) + } + diff := deep.Equal(mutatedPod, &v.mutatedPod) + if len(diff) > 0 { + t.Errorf("%s failed with mutated pod diff: %v", k, diff) + } + } +} diff --git a/release/.gitignore b/release/.gitignore deleted file mode 100644 index 303b66b5..00000000 --- a/release/.gitignore +++ /dev/null @@ -1 +0,0 @@ -kustomization.yaml diff --git a/release/admiralty/kustomization.tmpl.yaml b/release/admiralty/kustomization.tmpl.yaml deleted file mode 100644 index f6801c6d..00000000 --- a/release/admiralty/kustomization.tmpl.yaml +++ /dev/null @@ -1,7 +0,0 @@ -bases: -- ../../config/agent/admiralty -imageTags: -- name: quay.io/admiralty/multicluster-scheduler-agent - newTag: RELEASE -- name: quay.io/admiralty/multicluster-scheduler-pod-admission-controller - newTag: RELEASE diff --git a/release/multicluster-service-account/kustomization.tmpl.yaml b/release/multicluster-service-account/kustomization.tmpl.yaml deleted file mode 100644 index 2f764ae0..00000000 --- a/release/multicluster-service-account/kustomization.tmpl.yaml +++ /dev/null @@ -1,7 +0,0 @@ -bases: -- ../../config/agent/multicluster-service-account -imageTags: -- name: quay.io/admiralty/multicluster-scheduler-agent - newTag: RELEASE -- name: quay.io/admiralty/multicluster-scheduler-pod-admission-controller - newTag: RELEASE diff --git a/release/release.sh b/release/release.sh index 99fd75bc..afcfc2f3 100755 --- a/release/release.sh +++ b/release/release.sh @@ -1,16 +1,17 @@ +#!/usr/bin/env bash set -euo pipefail -RELEASE="$1" +VERSION="$1" -DEPLOYMENTS=("admiralty" "multicluster-service-account" "scheduler") -for DEPLOYMENT in "${DEPLOYMENTS[@]}"; do - sed "s/RELEASE/$RELEASE/g" "release/$DEPLOYMENT/kustomization.tmpl.yaml" > "release/$DEPLOYMENT/kustomization.yaml" -done +IMAGES=( + "multicluster-scheduler-agent" + "multicluster-scheduler-pod-admission-controller" + "multicluster-scheduler-basic" +) -kustomize build release/admiralty -o _out/admiralty.yaml -kustomize build release/multicluster-service-account -o _out/agent.yaml -kustomize build release/scheduler -o _out/scheduler.yaml -# TODO: upload to GitHub +for IMAGE in "${IMAGES[@]}"; do + docker push "quay.io/admiralty/$IMAGE:$VERSION" +done -RELEASE=$RELEASE skaffold build -f release/skaffold.yaml +# TODO: upload Helm chart # TODO: also tag images with latest diff --git a/release/scheduler/kustomization.tmpl.yaml b/release/scheduler/kustomization.tmpl.yaml deleted file mode 100644 index 67a0aa21..00000000 --- a/release/scheduler/kustomization.tmpl.yaml +++ /dev/null @@ -1,5 +0,0 @@ -bases: -- ../../config/scheduler -imageTags: -- name: quay.io/admiralty/multicluster-scheduler-basic - newTag: RELEASE diff --git a/release/skaffold.yaml b/release/skaffold.yaml deleted file mode 100644 index f5fa0928..00000000 --- a/release/skaffold.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: skaffold/v1alpha2 -kind: Config -build: - tagPolicy: - envTemplate: - template: "{{.IMAGE_NAME}}:{{.RELEASE}}" - artifacts: - - imageName: quay.io/admiralty/multicluster-scheduler-agent - workspace: _out - docker: - buildArgs: - target: cmd/agent - - imageName: quay.io/admiralty/multicluster-scheduler-pod-admission-controller - workspace: _out - docker: - buildArgs: - target: cmd/pod-admission-controller - - imageName: quay.io/admiralty/multicluster-scheduler-basic - workspace: _out - docker: - buildArgs: - target: cmd/scheduler -deploy: - kubectl: {} diff --git a/test/e2e/agent1/deployment_patch.yaml b/test/e2e/agent1/deployment_patch.yaml deleted file mode 100644 index 72a690e8..00000000 --- a/test/e2e/agent1/deployment_patch.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: multicluster-scheduler-agent -spec: - template: - metadata: - annotations: - multicluster.admiralty.io/service-account-import.name: cluster1 diff --git a/test/e2e/agent1/kustomization.yaml b/test/e2e/agent1/kustomization.yaml deleted file mode 100644 index 0510e85b..00000000 --- a/test/e2e/agent1/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -bases: -- ../../../config/agent/ -patches: -- deployment_patch.yaml -resources: -- serviceaccountimport.yaml diff --git a/test/e2e/agent1/serviceaccountimport.yaml b/test/e2e/agent1/serviceaccountimport.yaml deleted file mode 100644 index f1d817f3..00000000 --- a/test/e2e/agent1/serviceaccountimport.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: multicluster.admiralty.io/v1alpha1 -kind: ServiceAccountImport -metadata: - name: cluster1 - namespace: multicluster-scheduler-agent # TODO: inform kustomize of CRD so it can auto-populate namespace -spec: - clusterName: cluster1 - namespace: foo - name: cluster1 diff --git a/test/e2e/agent1/skaffold.yaml b/test/e2e/agent1/skaffold.yaml deleted file mode 100644 index 28169d41..00000000 --- a/test/e2e/agent1/skaffold.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: skaffold/v1alpha2 -kind: Config -build: - tagPolicy: - sha256: {} - artifacts: - - imageName: quay.io/admiralty/multicluster-scheduler-agent - workspace: _out - docker: - buildArgs: - target: cmd/agent - - imageName: quay.io/admiralty/multicluster-scheduler-pod-admission-controller - workspace: _out - docker: - buildArgs: - target: cmd/pod-admission-controller -deploy: - kustomize: - kustomizePath: test/e2e/agent1 diff --git a/test/e2e/agent2/deployment_patch.yaml b/test/e2e/agent2/deployment_patch.yaml deleted file mode 100644 index 0a47c481..00000000 --- a/test/e2e/agent2/deployment_patch.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: multicluster-scheduler-agent -spec: - template: - metadata: - annotations: - multicluster.admiralty.io/service-account-import.name: cluster2 diff --git a/test/e2e/agent2/kustomization.yaml b/test/e2e/agent2/kustomization.yaml deleted file mode 100644 index 0510e85b..00000000 --- a/test/e2e/agent2/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -bases: -- ../../../config/agent/ -patches: -- deployment_patch.yaml -resources: -- serviceaccountimport.yaml diff --git a/test/e2e/agent2/serviceaccountimport.yaml b/test/e2e/agent2/serviceaccountimport.yaml deleted file mode 100644 index a2acb2bb..00000000 --- a/test/e2e/agent2/serviceaccountimport.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: multicluster.admiralty.io/v1alpha1 -kind: ServiceAccountImport -metadata: - name: cluster2 - namespace: multicluster-scheduler-agent # TODO: inform kustomize of CRD so it can auto-populate namespace -spec: - clusterName: cluster1 - namespace: foo - name: cluster2 diff --git a/test/e2e/agent2/skaffold.yaml b/test/e2e/agent2/skaffold.yaml deleted file mode 100644 index ffadaff5..00000000 --- a/test/e2e/agent2/skaffold.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: skaffold/v1alpha2 -kind: Config -build: - tagPolicy: - sha256: {} - artifacts: - - imageName: quay.io/admiralty/multicluster-scheduler-agent - workspace: _out - docker: - buildArgs: - target: cmd/agent - - imageName: quay.io/admiralty/multicluster-scheduler-pod-admission-controller - workspace: _out - docker: - buildArgs: - target: cmd/pod-admission-controller -deploy: - kustomize: - kustomizePath: test/e2e/agent2 diff --git a/test/e2e/aliases.sh b/test/e2e/aliases.sh old mode 100644 new mode 100755 index 6a730ccf..e4d7a8fe --- a/test/e2e/aliases.sh +++ b/test/e2e/aliases.sh @@ -1,4 +1,6 @@ -k1() { kubectl --context cluster1 "$@"; } -k2() { kubectl --context cluster2 "$@"; } -c1() { kubectl config use-context cluster1 "$@"; } -c2() { kubectl config use-context cluster2 "$@"; } +k1() { KUBECONFIG=kubeconfig-cluster1 kubectl "$@"; } +k2() { KUBECONFIG=kubeconfig-cluster2 kubectl "$@"; } +k3() { KUBECONFIG=kubeconfig-cluster3 kubectl "$@"; } +helm1() { KUBECONFIG=kubeconfig-cluster1 helm "$@"; } +helm2() { KUBECONFIG=kubeconfig-cluster2 helm "$@"; } +helm3() { KUBECONFIG=kubeconfig-cluster3 helm "$@"; } diff --git a/test/e2e/cilium.sh b/test/e2e/cilium.sh deleted file mode 100755 index e6d0d5a4..00000000 --- a/test/e2e/cilium.sh +++ /dev/null @@ -1,135 +0,0 @@ -set -euo pipefail - -source test/e2e/aliases.sh - -setup() { - git clone https://github.com/cilium/clustermesh-tools.git - cd clustermesh-tools - export NAMESPACE=cilium # for extract-etcd-secrets.sh (default: kube-system) - - for CLUSTER_ID in 1 2; do - CLUSTER_NAME=cluster$CLUSTER_ID - - # from https://docs.cilium.io/en/v1.4/gettingstarted/k8s-install-gke/ - k$CLUSTER_ID create ns cilium - - k$CLUSTER_ID -n cilium apply -f https://raw.githubusercontent.com/cilium/cilium/v1.4.2/examples/kubernetes/node-init/node-init.yaml - - echo "waiting for daemon set cilium-node-init to be ready" - until [ $(k$CLUSTER_ID -n cilium get ds cilium-node-init -o jsonpath="{.status.numberReady}") == 3 ]; do echo -n "."; sleep 1; done; echo - - echo "waiting for cilium-node-init to have completed successfully on all nodes" # redundant? - until [ $(k$CLUSTER_ID -n cilium logs -l app=cilium-node-init 2>/dev/null | grep "startup-script succeeded" | wc -l) == 3 ]; do echo -n "."; sleep 1; done; echo - - echo "waiting for all nodes to be ready..." - k$CLUSTER_ID wait node --for condition=ready -l beta.kubernetes.io/arch=amd64 --timeout=60s - - k$CLUSTER_ID -n kube-system delete pod -l k8s-app=kube-dns - # note: kube-dns won't restart until cilium is available ("failed to find plugin cilium-cni") - # however, cilium etcd checks DNS - - k$CLUSTER_ID apply -f https://raw.githubusercontent.com/cilium/cilium/v1.4.2/examples/kubernetes/1.11/cilium-with-node-init.yaml - - echo "waiting for daemon set cilium to be ready" - until [ $(k$CLUSTER_ID -n cilium get ds cilium -o jsonpath="{.status.numberReady}") == 3 ]; do echo -n "."; sleep 1; done; echo - - echo "waiting for deployment cilium-operator to be available..." - k$CLUSTER_ID -n cilium wait deploy/cilium-operator --for condition=available --timeout=60s - - echo "waiting for deployment cilium-etcd-operator to be available..." - k$CLUSTER_ID -n cilium wait deploy/cilium-etcd-operator --for condition=available --timeout=60s - - echo "waiting for EtcdCluster cilium-etcd to be available..." - k$CLUSTER_ID -n cilium wait EtcdCluster/cilium-etcd --for condition=available --timeout=60s - - echo "waiting for deployment kube-dns to be available..." - k$CLUSTER_ID -n kube-system wait deploy/kube-dns --for condition=available --timeout=60s - - # restart pods stuck in crash loop (and prevent tear-down...) - k$CLUSTER_ID -n kube-system delete pod -l k8s-app=heapster - k$CLUSTER_ID -n kube-system delete pod -l k8s-app=glbc - - # from https://docs.cilium.io/en/v1.4/gettingstarted/clustermesh/ - k$CLUSTER_ID -n cilium patch cm cilium-config -p '{"data":{"cluster-id": "'$CLUSTER_ID'", "cluster-name": "'$CLUSTER_NAME'"}}' - k$CLUSTER_ID -n cilium apply -f https://raw.githubusercontent.com/cilium/cilium/v1.4.2/examples/kubernetes/clustermesh/cilium-etcd-external-service/cilium-etcd-external-gke.yaml - - echo "waiting for secret cilium-etcd-secrets to exist" - until k$CLUSTER_ID -n cilium get secret "cilium-etcd-secrets" &> /dev/null; do echo -n "."; sleep 1; done; echo - - echo "waiting for service cilium-etcd-external to have an load balancer ingress IP" - while [ -z "$(k$CLUSTER_ID -n cilium get svc cilium-etcd-external -o jsonpath="{.status.loadBalancer.ingress[0].ip}")" ]; do echo -n "."; sleep 1; done; echo - - c$CLUSTER_ID && ./extract-etcd-secrets.sh - done - - ./generate-secret-yaml.sh > clustermesh.yaml - ./generate-name-mapping.sh > ds.patch - - for CLUSTER_ID in 1 2; do - CLUSTER_NAME=cluster$CLUSTER_ID - - k$CLUSTER_ID -n cilium patch ds cilium -p "$(cat ds.patch)" - k$CLUSTER_ID -n cilium apply -f clustermesh.yaml - k$CLUSTER_ID -n cilium delete pod -l k8s-app=cilium - - echo "waiting for daemon set cilium to be ready" - until [ $(k$CLUSTER_ID -n cilium get ds cilium -o jsonpath="{.status.numberReady}") == 3 ]; do echo -n "."; sleep 1; done; echo - - # missing step in doc: cilium-operator must be restarted - # "the cilium-operator deployment [...] is responsible to propagate Kubernetes services into the kvstore" - # TODO: cilium/cilium PR - k$CLUSTER_ID -n cilium delete pod -l name=cilium-operator - - echo "waiting for deployment cilium-operator to be available..." - k$CLUSTER_ID -n cilium wait deploy/cilium-operator --for condition=available --timeout=60s - - k$CLUSTER_ID apply -f https://raw.githubusercontent.com/cilium/cilium/v1.4.2/examples/kubernetes/clustermesh/global-service-example/$CLUSTER_NAME.yaml - - echo "waiting for deployment rebel-base to roll out..." - k$CLUSTER_ID rollout status deploy/rebel-base - - echo "waiting for deployment x-wing to roll out..." - k$CLUSTER_ID rollout status deploy/x-wing - done - - cd .. - - rm -rf clustermesh-tools -} - -test() { - CLUSTER1_REBEL_BASE_POD_IP=$(k1 get pod -l name=rebel-base -o jsonpath={.items[0].status.podIP}) - echo "CLUSTER1_REBEL_BASE_POD_IP=$CLUSTER1_REBEL_BASE_POD_IP" - CLUSTER2_REBEL_BASE_POD_IP=$(k2 get pod -l name=rebel-base -o jsonpath={.items[0].status.podIP}) - echo "CLUSTER2_REBEL_BASE_POD_IP=$CLUSTER2_REBEL_BASE_POD_IP" - - CLUSTER1_X_WING_POD_NAME=$(k1 get pod -l name=x-wing -o jsonpath={.items[0].metadata.name}) - echo "CLUSTER1_X_WING_POD_NAME=$CLUSTER1_X_WING_POD_NAME" - CLUSTER2_X_WING_POD_NAME=$(k2 get pod -l name=x-wing -o jsonpath={.items[0].metadata.name}) - echo "CLUSTER2_X_WING_POD_NAME=$CLUSTER2_X_WING_POD_NAME" - - k1 exec $CLUSTER1_X_WING_POD_NAME -- curl -s $CLUSTER2_REBEL_BASE_POD_IP - k2 exec $CLUSTER2_X_WING_POD_NAME -- curl -s $CLUSTER1_REBEL_BASE_POD_IP - - echo "calling from cluster1" - Cluster="" - until [ "$Cluster" == "Cluster-2" ]; do - Cluster=$(k1 exec $CLUSTER1_X_WING_POD_NAME -- curl -s rebel-base | jq -r .Cluster) - echo $Cluster - done - - echo "calling from cluster2" - Cluster="" - until [ "$Cluster" == "Cluster-1" ]; do - Cluster=$(k2 exec $CLUSTER2_X_WING_POD_NAME -- curl -s rebel-base | jq -r .Cluster) - echo $Cluster - done -} - -# tear_down() { - -# } - -setup -test -# TODO: tear down diff --git a/test/e2e/cluster-namespaces/test.sh b/test/e2e/cluster-namespaces/test.sh new file mode 100755 index 00000000..4d62c641 --- /dev/null +++ b/test/e2e/cluster-namespaces/test.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail + +source test/e2e/aliases.sh + +setup() { + k1 create ns c1 + k1 create ns c2 + + k1 create ns multicluster-scheduler + k2 create ns multicluster-scheduler + + helm1 upgrade --install multicluster-scheduler charts/multicluster-scheduler -n multicluster-scheduler -f test/e2e/cluster-namespaces/values-cluster1.yaml + helm2 upgrade --install multicluster-scheduler charts/multicluster-scheduler -n multicluster-scheduler -f test/e2e/cluster-namespaces/values-cluster2.yaml + + ./kubemcsa export --kubeconfig kubeconfig-cluster1 member -n c1 --as remote | k1 apply -n multicluster-scheduler -f - + ./kubemcsa export --kubeconfig kubeconfig-cluster1 member -n c2 --as remote | k2 apply -n multicluster-scheduler -f - +} + +tear_down() { + helm1 delete multicluster-scheduler + helm2 delete multicluster-scheduler + + k1 delete secret remote -n multicluster-scheduler + k2 delete secret remote -n multicluster-scheduler + + k1 delete ns c1 + k1 delete ns c2 + + k1 delete ns multicluster-scheduler + k2 delete ns multicluster-scheduler +} diff --git a/test/e2e/cluster-namespaces/values-cluster1.yaml b/test/e2e/cluster-namespaces/values-cluster1.yaml new file mode 100644 index 00000000..1b4f8931 --- /dev/null +++ b/test/e2e/cluster-namespaces/values-cluster1.yaml @@ -0,0 +1,27 @@ +global: + useClusterNamespaces: true + clusters: + - name: c1 + clusterNamespace: c1 + - name: c2 + clusterNamespace: c2 + +scheduler: + enabled: true + securityContext: + runAsUser: 1000 + +agent: + enabled: true + remotes: + - secretName: remote + securityContext: + runAsUser: 1000 + +webhook: + enabled: true + securityContext: + runAsUser: 1000 + +clusters: + enabled: true diff --git a/test/e2e/cluster-namespaces/values-cluster2.yaml b/test/e2e/cluster-namespaces/values-cluster2.yaml new file mode 100644 index 00000000..1e9bdfc6 --- /dev/null +++ b/test/e2e/cluster-namespaces/values-cluster2.yaml @@ -0,0 +1,14 @@ +global: + useClusterNamespaces: true + +agent: + enabled: true + remotes: + - secretName: remote + securityContext: + runAsUser: 1000 + +webhook: + enabled: true + securityContext: + runAsUser: 1000 diff --git a/test/e2e/clusters.sh b/test/e2e/clusters.sh new file mode 100755 index 00000000..28ed026a --- /dev/null +++ b/test/e2e/clusters.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail + +DIR=$(dirname "$0") + +VERSION="$1" + +setup_clusters() { + for CLUSTER in cluster1 cluster2 cluster3; do + kind create cluster --name $CLUSTER --wait 5m + kind get kubeconfig --name $CLUSTER --internal >kubeconfig-$CLUSTER + KUBECONFIG=kubeconfig-$CLUSTER kubectl apply -f "$DIR"/must-run-as-non-root.yaml + + kind load docker-image "quay.io/admiralty/multicluster-scheduler-basic:$VERSION" --name $CLUSTER + kind load docker-image "quay.io/admiralty/multicluster-scheduler-agent:$VERSION" --name $CLUSTER + kind load docker-image "quay.io/admiralty/multicluster-scheduler-pod-admission-controller:$VERSION" --name $CLUSTER + done +} + +tear_down_clusters() { + for CLUSTER in cluster1 cluster2 cluster3; do + rm -f kubeconfig-$CLUSTER + kind delete cluster --name $CLUSTER # if exists + done +} diff --git a/test/e2e/e2e.sh b/test/e2e/e2e.sh index 3dd1b4f8..05a95bb0 100755 --- a/test/e2e/e2e.sh +++ b/test/e2e/e2e.sh @@ -1,9 +1,21 @@ +#!/usr/bin/env bash set -euo pipefail -test/e2e/setup_clusters.sh -test/e2e/cilium.sh -test/e2e/setup.sh -test/e2e/test_networking.sh -test/e2e/test_argo.sh -test/e2e/tear_down.sh -test/e2e/tear_down_clusters.sh +VERSION="$1" + +source test/e2e/mcsa.sh +install_kubemcsa + +source test/e2e/clusters.sh $VERSION +source test/e2e/test_argo.sh + +for T in single-namespace cluster-namespaces with-mcsa; do # TODO multi-federation + setup_clusters + setup_argo + source test/e2e/$T/test.sh + setup + test_blog_scenario_a_multicluster + tear_down + tear_down_argo + tear_down_clusters +done diff --git a/test/e2e/federation/kustomization.yaml b/test/e2e/federation/kustomization.yaml deleted file mode 100644 index 5ee1789c..00000000 --- a/test/e2e/federation/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -namespace: foo -resources: -- namespace.yaml -- rbac.yaml diff --git a/test/e2e/federation/namespace.yaml b/test/e2e/federation/namespace.yaml deleted file mode 100644 index be0f4fd1..00000000 --- a/test/e2e/federation/namespace.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: foo diff --git a/test/e2e/federation/rbac.yaml b/test/e2e/federation/rbac.yaml deleted file mode 100644 index 5776f99c..00000000 --- a/test/e2e/federation/rbac.yaml +++ /dev/null @@ -1,33 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: cluster1 ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: cluster1 -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: multicluster-scheduler-member -subjects: -- kind: ServiceAccount - name: cluster1 ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: cluster2 ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: cluster2 -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: multicluster-scheduler-member -subjects: -- kind: ServiceAccount - name: cluster2 diff --git a/test/e2e/mcsa.sh b/test/e2e/mcsa.sh new file mode 100644 index 00000000..126c924a --- /dev/null +++ b/test/e2e/mcsa.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail + +MCSA_RELEASE_URL=https://github.com/admiraltyio/multicluster-service-account/releases/download/v0.6.1 + +install_kubemcsa() { + OS=linux + ARCH=amd64 + + curl -Lo kubemcsa "$MCSA_RELEASE_URL/kubemcsa-$OS-$ARCH" + chmod +x kubemcsa +} diff --git a/test/e2e/must-run-as-non-root.yaml b/test/e2e/must-run-as-non-root.yaml new file mode 100644 index 00000000..a831df42 --- /dev/null +++ b/test/e2e/must-run-as-non-root.yaml @@ -0,0 +1,46 @@ +--- +apiVersion: extensions/v1beta1 +kind: PodSecurityPolicy +metadata: + name: must-run-as-non-root +spec: + fsGroup: + rule: RunAsAny + runAsUser: + rule: MustRunAsNonRoot + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - '*' +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: must-run-as-non-root +rules: + - apiGroups: + - extensions + resources: + - podsecuritypolicies + resourceNames: + - must-run-as-non-root + verbs: + - use +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: must-run-as-non-root +subjects: + - kind: Group + name: system:authenticated + apiGroup: rbac.authorization.k8s.io + - kind: Group + name: system:serviceaccounts + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: ClusterRole + name: must-run-as-non-root + apiGroup: rbac.authorization.k8s.io diff --git a/test/e2e/networking/cluster1/kubernetes.yaml b/test/e2e/networking/cluster1/kubernetes.yaml deleted file mode 100644 index de37dad5..00000000 --- a/test/e2e/networking/cluster1/kubernetes.yaml +++ /dev/null @@ -1,143 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: sample-service-cluster1 -spec: - selector: - matchLabels: - app: sample-service-cluster1 - template: - metadata: - labels: - app: sample-service-cluster1 - spec: - containers: - - image: quay.io/admiralty/sample-service - name: sample-service - ports: - - containerPort: 80 - resources: - requests: - cpu: 50m ---- -kind: Service -apiVersion: v1 -metadata: - name: sample-service-cluster1 -spec: - selector: - app: sample-service-cluster1 - ports: - - protocol: TCP - port: 80 - targetPort: 80 ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: global-sample-service-cluster1 -spec: - selector: - matchLabels: - app: global-sample-service-cluster1 - template: - metadata: - labels: - app: global-sample-service-cluster1 - spec: - containers: - - image: quay.io/admiralty/sample-service - name: sample-service - ports: - - containerPort: 80 - resources: - requests: - cpu: 50m ---- -kind: Service -apiVersion: v1 -metadata: - name: global-sample-service-cluster1 - annotations: - io.cilium/global-service: "true" -spec: - selector: - app: global-sample-service-cluster1 - ports: - - protocol: TCP - port: 80 - targetPort: 80 ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: mc-sample-service-cluster1 -spec: - selector: - matchLabels: - app: mc-sample-service-cluster1 - template: - metadata: - labels: - app: mc-sample-service-cluster1 - annotations: - multicluster.admiralty.io/elect: "" - multicluster.admiralty.io/clustername: "cluster1" - spec: - containers: - - image: quay.io/admiralty/sample-service - name: sample-service - ports: - - containerPort: 80 - resources: - requests: - cpu: 50m ---- -kind: Service -apiVersion: v1 -metadata: - name: mc-sample-service-cluster1 -spec: - selector: - app: mc-sample-service-cluster1 - ports: - - protocol: TCP - port: 80 - targetPort: 80 ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: mc-sample-service-cluster2 -spec: - selector: - matchLabels: - app: mc-sample-service-cluster2 - template: - metadata: - labels: - app: mc-sample-service-cluster2 - annotations: - multicluster.admiralty.io/elect: "" - multicluster.admiralty.io/clustername: "cluster2" - spec: - containers: - - image: quay.io/admiralty/sample-service - name: sample-service - ports: - - containerPort: 80 - resources: - requests: - cpu: 50m ---- -kind: Service -apiVersion: v1 -metadata: - name: mc-sample-service-cluster2 -spec: - selector: - app: mc-sample-service-cluster2 - ports: - - protocol: TCP - port: 80 - targetPort: 80 diff --git a/test/e2e/networking/cluster1/skaffold.yaml b/test/e2e/networking/cluster1/skaffold.yaml deleted file mode 100644 index 9312d754..00000000 --- a/test/e2e/networking/cluster1/skaffold.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: skaffold/v1alpha2 -kind: Config -build: - tagPolicy: - sha256: {} - artifacts: - - imageName: quay.io/admiralty/sample-service - workspace: test/e2e/sample-service - docker: {} -deploy: - kubectl: - manifests: - - test/e2e/networking/cluster1/kubernetes.yaml diff --git a/test/e2e/networking/cluster2/kubernetes.yaml b/test/e2e/networking/cluster2/kubernetes.yaml deleted file mode 100644 index dd55559d..00000000 --- a/test/e2e/networking/cluster2/kubernetes.yaml +++ /dev/null @@ -1,69 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: sample-service-cluster2 -spec: - selector: - matchLabels: - app: sample-service-cluster2 - template: - metadata: - labels: - app: sample-service-cluster2 - spec: - containers: - - image: quay.io/admiralty/sample-service - name: sample-service - ports: - - containerPort: 80 - resources: - requests: - cpu: 50m ---- -kind: Service -apiVersion: v1 -metadata: - name: sample-service-cluster2 -spec: - selector: - app: sample-service-cluster2 - ports: - - protocol: TCP - port: 80 - targetPort: 80 ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: global-sample-service-cluster2 -spec: - selector: - matchLabels: - app: global-sample-service-cluster2 - template: - metadata: - labels: - app: global-sample-service-cluster2 - spec: - containers: - - image: quay.io/admiralty/sample-service - name: sample-service - ports: - - containerPort: 80 - resources: - requests: - cpu: 50m ---- -kind: Service -apiVersion: v1 -metadata: - name: global-sample-service-cluster2 - annotations: - io.cilium/global-service: "true" -spec: - selector: - app: global-sample-service-cluster2 - ports: - - protocol: TCP - port: 80 - targetPort: 80 diff --git a/test/e2e/networking/cluster2/skaffold.yaml b/test/e2e/networking/cluster2/skaffold.yaml deleted file mode 100644 index 81f01728..00000000 --- a/test/e2e/networking/cluster2/skaffold.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: skaffold/v1alpha2 -kind: Config -build: - tagPolicy: - sha256: {} - artifacts: - - imageName: quay.io/admiralty/sample-service - workspace: test/e2e/sample-service - docker: {} -deploy: - kubectl: - manifests: - - test/e2e/networking/cluster2/kubernetes.yaml diff --git a/test/e2e/sample-service/Dockerfile b/test/e2e/sample-service/Dockerfile deleted file mode 100644 index 2aef8fe5..00000000 --- a/test/e2e/sample-service/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM python:3-alpine -RUN apk --no-cache add curl -RUN pip install --no-cache-dir pipenv -COPY Pipfile Pipfile.lock ./ -RUN pipenv install --system --deploy -COPY app.py ./ -ENV FLASK_APP=app.py -ENV FLASK_ENV=development -ENTRYPOINT [ "flask", "run", "--host", "0.0.0.0", "--port", "80" ] diff --git a/test/e2e/sample-service/Pipfile b/test/e2e/sample-service/Pipfile deleted file mode 100644 index 4fce346a..00000000 --- a/test/e2e/sample-service/Pipfile +++ /dev/null @@ -1,13 +0,0 @@ -[[source]] -name = "pypi" -url = "https://pypi.org/simple" -verify_ssl = true - -[dev-packages] - -[packages] -flask = "*" -requests = "*" - -[requires] -python_version = "3.7" diff --git a/test/e2e/sample-service/Pipfile.lock b/test/e2e/sample-service/Pipfile.lock deleted file mode 100644 index a52b3528..00000000 --- a/test/e2e/sample-service/Pipfile.lock +++ /dev/null @@ -1,127 +0,0 @@ -{ - "_meta": { - "hash": { - "sha256": "d99bd6e13b173278b5b31022309611c275578426d46751b984b34b7c6f9234af" - }, - "pipfile-spec": 6, - "requires": { - "python_version": "3.7" - }, - "sources": [ - { - "name": "pypi", - "url": "https://pypi.org/simple", - "verify_ssl": true - } - ] - }, - "default": { - "certifi": { - "hashes": [ - "sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50", - "sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef" - ], - "version": "==2019.9.11" - }, - "chardet": { - "hashes": [ - "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", - "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" - ], - "version": "==3.0.4" - }, - "click": { - "hashes": [ - "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", - "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7" - ], - "version": "==7.0" - }, - "flask": { - "hashes": [ - "sha256:2271c0070dbcb5275fad4a82e29f23ab92682dc45f9dfbc22c02ba9b9322ce48", - "sha256:a080b744b7e345ccfcbc77954861cb05b3c63786e93f2b3875e0913d44b43f05" - ], - "index": "pypi", - "version": "==1.0.2" - }, - "idna": { - "hashes": [ - "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", - "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" - ], - "version": "==2.8" - }, - "itsdangerous": { - "hashes": [ - "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19", - "sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749" - ], - "version": "==1.1.0" - }, - "jinja2": { - "hashes": [ - "sha256:74320bb91f31270f9551d46522e33af46a80c3d619f4a4bf42b3164d30b5911f", - "sha256:9fe95f19286cfefaa917656583d020be14e7859c6b0252588391e47db34527de" - ], - "version": "==2.10.3" - }, - "markupsafe": { - "hashes": [ - "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", - "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161", - "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235", - "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5", - "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff", - "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b", - "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1", - "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e", - "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183", - "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66", - "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1", - "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1", - "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e", - "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b", - "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905", - "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735", - "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d", - "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e", - "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d", - "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c", - "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21", - "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2", - "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5", - "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b", - "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6", - "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f", - "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f", - "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7" - ], - "version": "==1.1.1" - }, - "requests": { - "hashes": [ - "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e", - "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b" - ], - "index": "pypi", - "version": "==2.21.0" - }, - "urllib3": { - "hashes": [ - "sha256:2393a695cd12afedd0dcb26fe5d50d0cf248e5a66f75dbd89a3d4eb333a61af4", - "sha256:a637e5fae88995b256e3409dc4d52c2e2e0ba32c42a6365fee8bbd2238de3cfb" - ], - "version": "==1.24.3" - }, - "werkzeug": { - "hashes": [ - "sha256:97660b282aa7e29f94f3fe378e5c7162d7ab9d601a8dbb1cbb2ffc8f0e54607d", - "sha256:cfd1281b1748288e59762c0e174d64d8bcb2b70e7c57bc4a1203c8825af24ac3" - ], - "index": "pypi", - "version": "==0.15.3" - } - }, - "develop": {} -} diff --git a/test/e2e/sample-service/app.py b/test/e2e/sample-service/app.py deleted file mode 100644 index 359b0a1d..00000000 --- a/test/e2e/sample-service/app.py +++ /dev/null @@ -1,10 +0,0 @@ -import socket - -import flask - -app = flask.Flask(__name__) - - -@app.route('/') -def hello_world(): - return socket.gethostname() diff --git a/test/e2e/scheduler/skaffold.yaml b/test/e2e/scheduler/skaffold.yaml deleted file mode 100644 index 875c04c6..00000000 --- a/test/e2e/scheduler/skaffold.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: skaffold/v1alpha2 -kind: Config -build: - tagPolicy: - sha256: {} - artifacts: - - imageName: quay.io/admiralty/multicluster-scheduler-basic - workspace: _out - docker: - buildArgs: - target: cmd/scheduler -deploy: - kustomize: - kustomizePath: config/scheduler diff --git a/test/e2e/setup.sh b/test/e2e/setup.sh deleted file mode 100755 index 5f08bd38..00000000 --- a/test/e2e/setup.sh +++ /dev/null @@ -1,30 +0,0 @@ -set -euo pipefail - -source test/e2e/aliases.sh - -install_bootstrap_multicluster_service_account() { - # Install MCSA and bootstrap cluster1 and cluster2 to import from cluster1 (which will host the control plane) - RELEASE_URL=https://github.com/admiraltyio/multicluster-service-account/releases/download/v0.3.1 - MCSA_URL="$RELEASE_URL/install.yaml" - k1 apply -f "$MCSA_URL" - k2 apply -f "$MCSA_URL" - # TODO: don't assume the right kubemcsa is installed - kubemcsa bootstrap cluster1 cluster1 - kubemcsa bootstrap cluster2 cluster1 -} - -install_multicluster_scheduler() { - c1 && skaffold run -f test/e2e/scheduler/skaffold.yaml - kustomize build test/e2e/federation | k1 apply -f - - c1 && skaffold run -f test/e2e/agent1/skaffold.yaml - - c2 && skaffold run -f test/e2e/agent2/skaffold.yaml - # TODO: skaffold deploy rather than run, because images have already been built for cluster1 (need to pass tagged image names) - - # switch back to cluster1 for user commands afterward - # TODO: save current context at beginning and return to it - c1 -} - -install_bootstrap_multicluster_service_account -install_multicluster_scheduler diff --git a/test/e2e/setup_clusters.sh b/test/e2e/setup_clusters.sh deleted file mode 100755 index 99d40b06..00000000 --- a/test/e2e/setup_clusters.sh +++ /dev/null @@ -1,13 +0,0 @@ -set -euo pipefail - -# Create cluster1 and cluster2 -PROJECT=$(gcloud config get-value project) -REGION=$(gcloud config get-value compute/zone) -for NAME in cluster1 cluster2; do - gcloud container clusters create $NAME --preemptible --enable-ip-alias - gcloud container clusters get-credentials $NAME - CONTEXT=gke_$PROJECT"_"$REGION"_"$NAME - sed -i -e "s/$CONTEXT/$NAME/g" ~/.kube/config - kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user $(gcloud config get-value account) - kubectl cluster-info -done diff --git a/test/e2e/single-namespace/test.sh b/test/e2e/single-namespace/test.sh new file mode 100755 index 00000000..20756dec --- /dev/null +++ b/test/e2e/single-namespace/test.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail + +source test/e2e/aliases.sh + +setup() { + helm1 upgrade --install multicluster-scheduler charts/multicluster-scheduler -f test/e2e/single-namespace/values-cluster1.yaml + helm2 upgrade --install multicluster-scheduler charts/multicluster-scheduler -f test/e2e/single-namespace/values-cluster2.yaml + + ./kubemcsa export --kubeconfig kubeconfig-cluster1 c1 --as remote | k1 apply -f - + ./kubemcsa export --kubeconfig kubeconfig-cluster1 c2 --as remote | k2 apply -f - +} + +tear_down() { + helm1 delete multicluster-scheduler + helm2 delete multicluster-scheduler + + k1 delete secret remote + k2 delete secret remote +} diff --git a/test/e2e/single-namespace/values-cluster1.yaml b/test/e2e/single-namespace/values-cluster1.yaml new file mode 100644 index 00000000..70c86d77 --- /dev/null +++ b/test/e2e/single-namespace/values-cluster1.yaml @@ -0,0 +1,25 @@ +global: + clusters: + - name: c1 + - name: c2 + +scheduler: + enabled: true + securityContext: + runAsUser: 1000 + +agent: + enabled: true + clusterName: c1 + remotes: + - secretName: remote + securityContext: + runAsUser: 1000 + +webhook: + enabled: true + securityContext: + runAsUser: 1000 + +clusters: + enabled: true diff --git a/test/e2e/single-namespace/values-cluster2.yaml b/test/e2e/single-namespace/values-cluster2.yaml new file mode 100644 index 00000000..dbe41cfe --- /dev/null +++ b/test/e2e/single-namespace/values-cluster2.yaml @@ -0,0 +1,12 @@ +agent: + enabled: true + clusterName: c2 + remotes: + - secretName: remote + securityContext: + runAsUser: 1000 + +webhook: + enabled: true + securityContext: + runAsUser: 1000 diff --git a/test/e2e/tear_down.sh b/test/e2e/tear_down.sh deleted file mode 100755 index 62241f07..00000000 --- a/test/e2e/tear_down.sh +++ /dev/null @@ -1,13 +0,0 @@ -# set -euo pipefail - -source test/e2e/aliases.sh - -c2 && skaffold delete -f test/e2e/agent2/skaffold.yaml -c1 && skaffold delete -f test/e2e/agent1/skaffold.yaml -kustomize build test/e2e/federation | k1 delete -f - -c1 && skaffold delete -f test/e2e/scheduler/skaffold.yaml - -RELEASE_URL=https://github.com/admiraltyio/multicluster-service-account/releases/download/v0.3.1 -MCSA_URL="$RELEASE_URL/install.yaml" -k2 delete -f "$MCSA_URL" -k1 delete -f "$MCSA_URL" diff --git a/test/e2e/tear_down_clusters.sh b/test/e2e/tear_down_clusters.sh deleted file mode 100755 index f5b9aadc..00000000 --- a/test/e2e/tear_down_clusters.sh +++ /dev/null @@ -1,8 +0,0 @@ -set -euo pipefail - -for NAME in cluster1 cluster2; do - gcloud container clusters delete $NAME --quiet - kubectl config delete-cluster $NAME - kubectl config delete-context $NAME - kubectl config unset users.$NAME -done diff --git a/test/e2e/test_argo.sh b/test/e2e/test_argo.sh index fb1ddf0d..77777385 100755 --- a/test/e2e/test_argo.sh +++ b/test/e2e/test_argo.sh @@ -2,41 +2,50 @@ set -euo pipefail source test/e2e/aliases.sh -setup() { - # Install Argo in cluster1 - k1 create ns argo - k1 apply -n argo -f https://raw.githubusercontent.com/argoproj/argo/v2.2.1/manifests/install.yaml - k1 apply -f config/samples/argo-workflows/_service-account.yaml - # the workflow service account must exist in the other cluster - k2 apply -f config/samples/argo-workflows/_service-account.yaml - - k1 label ns default multicluster-scheduler=enabled +setup_argo() { + # Install Argo in cluster1 + k1 create ns argo + k1 apply -n argo -f https://raw.githubusercontent.com/argoproj/argo/v2.2.1/manifests/install.yaml + + # kind uses containerd not docker so we change the argo executor (default: docker) + # TODO modify install.yaml instead + k1 patch cm -n argo workflow-controller-configmap --patch '{"data":{"config":"{\"containerRuntimeExecutor\":\"kubelet\"}"}}' + k1 delete pod --all -n argo # reload config map + + k1 apply -f examples/argo-workflows/_service-account.yaml + # the workflow service account must exist in the other cluster + k2 apply -f examples/argo-workflows/_service-account.yaml + + k1 label ns default multicluster-scheduler=enabled + + # TODO download only if not present or version mismatch + curl -Lo argo https://github.com/argoproj/argo/releases/download/v2.2.1/argo-linux-amd64 + chmod +x argo + + # speed up container creations + docker pull argoproj/argoexec:v2.2.1 # may already be on host + kind load docker-image argoproj/argoexec:v2.2.1 --name cluster1 + kind load docker-image argoproj/argoexec:v2.2.1 --name cluster2 } -tear_down() { - k1 label ns default multicluster-scheduler- +tear_down_argo() { + k1 label ns default multicluster-scheduler- - k2 delete -f config/samples/argo-workflows/_service-account.yaml - k1 delete -f config/samples/argo-workflows/_service-account.yaml - k1 delete -n argo -f https://raw.githubusercontent.com/argoproj/argo/v2.2.1/manifests/install.yaml - k1 delete ns argo + k2 delete -f examples/argo-workflows/_service-account.yaml + k1 delete -f examples/argo-workflows/_service-account.yaml + k1 delete -n argo -f https://raw.githubusercontent.com/argoproj/argo/v2.2.1/manifests/install.yaml + k1 delete ns argo } test_blog_scenario_a_multicluster() { - argo --context cluster1 submit --serviceaccount argo-workflow --watch config/samples/argo-workflows/blog-scenario-a-multicluster.yaml - - if [ $(k2 get pod | wc -l) -gt 1 ] - then - echo "SUCCESS" - exit 0 - else - echo "FAILURE" - exit 1 - fi - - argo --context cluster1 delete --all -} + KUBECONFIG=kubeconfig-cluster1 ./argo submit --serviceaccount argo-workflow --watch examples/argo-workflows/blog-scenario-a-multicluster.yaml + + if [ $(k2 get pod | wc -l) -gt 1 ]; then + echo "SUCCESS" + else + echo "FAILURE" + exit 1 + fi -setup -test_blog_scenario_a_multicluster -tear_down + KUBECONFIG=kubeconfig-cluster1 ./argo delete --all +} diff --git a/test/e2e/test_networking.sh b/test/e2e/test_networking.sh deleted file mode 100755 index e5b74cc2..00000000 --- a/test/e2e/test_networking.sh +++ /dev/null @@ -1,75 +0,0 @@ -set -euo pipefail - -source test/e2e/aliases.sh - -setup() { - k1 label ns default multicluster-scheduler=enabled - - c1 && skaffold run -f test/e2e/networking/cluster1/skaffold.yaml - c2 && skaffold run -f test/e2e/networking/cluster2/skaffold.yaml - - k1 rollout status deploy/global-sample-service-cluster1 - k1 rollout status deploy/mc-sample-service-cluster1 - k1 rollout status deploy/mc-sample-service-cluster2 - k1 rollout status deploy/sample-service-cluster1 - - k2 rollout status deploy/global-sample-service-cluster2 - k2 rollout status deploy/sample-service-cluster2 -} - -test() { - CLUSTER1_DELEGATE_MC_SAMPLE_SERVICE_POD_NAME=$(k1 get pod -l multicluster.admiralty.io/app=mc-sample-service-cluster1 -o jsonpath={.items[0].metadata.name}) - echo "CLUSTER1_DELEGATE_MC_SAMPLE_SERVICE_POD_NAME=$CLUSTER1_DELEGATE_MC_SAMPLE_SERVICE_POD_NAME" - - CLUSTER1_GLOBAL_SAMPLE_SERVICE_POD_NAME=$(k1 get pod -l app=global-sample-service-cluster1 -o jsonpath={.items[0].metadata.name}) - echo "CLUSTER1_GLOBAL_SAMPLE_SERVICE_POD_NAME=$CLUSTER1_GLOBAL_SAMPLE_SERVICE_POD_NAME" - - CLUSTER1_SAMPLE_SERVICE_POD_NAME=$(k1 get pod -l app=sample-service-cluster1 -o jsonpath={.items[0].metadata.name}) - echo "CLUSTER1_SAMPLE_SERVICE_POD_NAME=$CLUSTER1_SAMPLE_SERVICE_POD_NAME" - - CLUSTER2_DELEGATE_MC_SAMPLE_SERVICE_POD_NAME=$(k2 get pod -l multicluster.admiralty.io/app=mc-sample-service-cluster2 -o jsonpath={.items[0].metadata.name}) - echo "CLUSTER2_DELEGATE_MC_SAMPLE_SERVICE_POD_NAME=$CLUSTER2_DELEGATE_MC_SAMPLE_SERVICE_POD_NAME" - - CLUSTER2_GLOBAL_SAMPLE_SERVICE_POD_NAME=$(k2 get pod -l app=global-sample-service-cluster2 -o jsonpath={.items[0].metadata.name}) - echo "CLUSTER2_GLOBAL_SAMPLE_SERVICE_POD_NAME=$CLUSTER2_GLOBAL_SAMPLE_SERVICE_POD_NAME" - - CLUSTER2_SAMPLE_SERVICE_POD_NAME=$(k2 get pod -l app=sample-service-cluster2 -o jsonpath={.items[0].metadata.name}) - echo "CLUSTER2_SAMPLE_SERVICE_POD_NAME=$CLUSTER2_SAMPLE_SERVICE_POD_NAME" - - for pod_name in "$CLUSTER1_DELEGATE_MC_SAMPLE_SERVICE_POD_NAME" "$CLUSTER1_GLOBAL_SAMPLE_SERVICE_POD_NAME" "$CLUSTER1_SAMPLE_SERVICE_POD_NAME"; do - [ "$(k1 exec $pod_name -- curl -s global-sample-service-cluster1)" == "$CLUSTER1_GLOBAL_SAMPLE_SERVICE_POD_NAME" ] - echo $? - [ "$(k1 exec $pod_name -- curl -s global-sample-service-cluster2)" == "$CLUSTER2_GLOBAL_SAMPLE_SERVICE_POD_NAME" ] - echo $? - [ "$(k1 exec $pod_name -- curl -s mc-sample-service-cluster1)" == "$CLUSTER1_DELEGATE_MC_SAMPLE_SERVICE_POD_NAME" ] - echo $? - [ "$(k1 exec $pod_name -- curl -s mc-sample-service-cluster2)" == "$CLUSTER2_DELEGATE_MC_SAMPLE_SERVICE_POD_NAME" ] - echo $? - [ "$(k1 exec $pod_name -- curl -s sample-service-cluster1)" == "$CLUSTER1_SAMPLE_SERVICE_POD_NAME" ] - echo $? - done - - for pod_name in "$CLUSTER2_DELEGATE_MC_SAMPLE_SERVICE_POD_NAME" "$CLUSTER2_GLOBAL_SAMPLE_SERVICE_POD_NAME" "$CLUSTER2_SAMPLE_SERVICE_POD_NAME"; do - [ "$(k2 exec $pod_name -- curl -s global-sample-service-cluster1)" == "$CLUSTER1_GLOBAL_SAMPLE_SERVICE_POD_NAME" ] - echo $? - [ "$(k2 exec $pod_name -- curl -s global-sample-service-cluster2)" == "$CLUSTER2_GLOBAL_SAMPLE_SERVICE_POD_NAME" ] - echo $? - [ "$(k2 exec $pod_name -- curl -s mc-sample-service-cluster1)" == "$CLUSTER1_DELEGATE_MC_SAMPLE_SERVICE_POD_NAME" ] - echo $? - [ "$(k2 exec $pod_name -- curl -s mc-sample-service-cluster2)" == "$CLUSTER2_DELEGATE_MC_SAMPLE_SERVICE_POD_NAME" ] - echo $? - [ "$(k2 exec $pod_name -- curl -s sample-service-cluster2)" == "$CLUSTER2_SAMPLE_SERVICE_POD_NAME" ] - echo $? - done -} - -tear_down() { - c1 && skaffold delete -f test/e2e/networking/cluster1/skaffold.yaml - c2 && skaffold delete -f test/e2e/networking/cluster2/skaffold.yaml - - k1 label ns default multicluster-scheduler- -} - -setup -test -tear_down diff --git a/test/e2e/with-mcsa/test.sh b/test/e2e/with-mcsa/test.sh new file mode 100755 index 00000000..a488e728 --- /dev/null +++ b/test/e2e/with-mcsa/test.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +set -euo pipefail + +source test/e2e/aliases.sh +source test/e2e/mcsa.sh + +MCSA_URL="$MCSA_RELEASE_URL/install.yaml" + +setup() { + # Install MCSA and bootstrap cluster1 and cluster2 to import from cluster1 (which will host the control plane) + k1 apply -f "$MCSA_URL" + k2 apply -f "$MCSA_URL" + + ./kubemcsa bootstrap --target-kubeconfig kubeconfig-cluster1 --source-kubeconfig kubeconfig-cluster1 + ./kubemcsa bootstrap --target-kubeconfig kubeconfig-cluster2 --source-kubeconfig kubeconfig-cluster1 + + k1 label ns default multicluster-service-account=enabled --overwrite + k2 label ns default multicluster-service-account=enabled --overwrite + + helm1 upgrade --install multicluster-scheduler charts/multicluster-scheduler -f test/e2e/with-mcsa/values-cluster1.yaml + helm2 upgrade --install multicluster-scheduler charts/multicluster-scheduler -f test/e2e/with-mcsa/values-cluster2.yaml + + cat <