NPT-684 DM Upgrade Integration #107

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft

PavitraS wants to merge 51 commits into main from dm-upgrade-integration

common-library/pkg/nexus-compare/compare.go

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -17,18 +17,19 @@ import (
  
    )

    const (

    	SpecMatch     = ".*properties\\/spec.*"

    	StatusMatch   = ".*properties\\/status.*"

    	SingletonPath = "/is_singleton"

    	ApiGenPath    = "/nexus-rest-api-gen"

    	SpecMatch      = ".*properties\\/spec.*"

    	StatusMatch    = ".*properties\\/status.*"

    	SingletonPath  = "/is_singleton"

    	ApiGenPath     = "/nexus-rest-api-gen"

    	RequiredString = "required"

    )

    func CompareFiles(data1, data2 []byte) (bool, *bytes.Buffer, error) {

    	buffer := new(bytes.Buffer)

    	headerColor, _ := colorful.Hex("#B9311B")

    	fileColor, _ := colorful.Hex("#088F8F")

    	name, err := getSpecName(data1)

    	name, err := GetSpecName(data1)

    	if err != nil {

    		return true, nil, err

    	}

    @@ -163,8 +164,10 @@ func filterReport(r *dyff.Report) *dyff.Report {
  
    	for _, di := range r.Diffs {

    		var ds []dyff.Detail

    		for _, d := range di.Details {

    			if d.From == nil && d.To != nil { // allow to add stuff

    				continue

    			if d.From == nil && d.To != nil && !strings.HasSuffix(di.Path.String(), fmt.Sprintf("/%s", RequiredString)) { // allow to add stuff, but not in required list

    				if len(d.To.Content) == 0 || d.To.Content[0].Value != RequiredString { //allow to add required map field

    					continue

    				}

    			}

    			if di.Path.String() == SingletonPath && d.From.Value == strconv.FormatBool(true) && d.To.Value == strconv.FormatBool(false) { // allow singleton change to false

    				continue

    @@ -186,11 +189,11 @@ func filterReport(r *dyff.Report) *dyff.Report {
  
    }

    func getAnnotationReport(data1, data2 []byte) (dyff.Report, error) {

    	aNexus, err := getMapNode(data1, []string{"metadata", "annotations", "nexus"})

    	aNexus, err := GetMapNode(data1, []string{"metadata", "annotations", "nexus"})

    	if err != nil {

    		return dyff.Report{}, err

    	}

    	bNexus, err := getMapNode(data2, []string{"metadata", "annotations", "nexus"})

    	bNexus, err := GetMapNode(data2, []string{"metadata", "annotations", "nexus"})

    	if err != nil {

    		return dyff.Report{}, err

    	}

    @@ -246,7 +249,7 @@ func writeTempFile(data []byte) (*os.File, error) {
  
    	return aFile, err

    }

    func getSpecName(data []byte) (string, error) {

    func GetSpecName(data []byte) (string, error) {

    	t := make(map[string]interface{})

    	err := yaml.Unmarshal(data, &t)

    	if err != nil {

    @@ -255,7 +258,7 @@ func getSpecName(data []byte) (string, error) {
  
    	return t["metadata"].(map[string]interface{})["name"].(string), nil

    }

    func getMapNode(data []byte, path []string) (interface{}, error) {

    func GetMapNode(data []byte, path []string) (interface{}, error) {

    	var t interface{}

    	var ok bool

    	errPath := "root"

common-library/pkg/nexus-compare/compare_test.go

-Original file line number
+Diff line change
@@ Expand Up / @@ -30,11 +30,18 @@ var _ = Describe("Compare lib tests", func() { @@
     			Expect(text.String()).Should(ContainSubstring(v))
     		}
     	})
-    	It("should return false when adding field", func() {
+    	It("should return false and only notice change in required", func() {
     		ans, text, err := CompareFiles([]byte(baseSpec), []byte(addedField))
     		Expect(err).NotTo(HaveOccurred())
-    		Expect(ans).To(BeFalse())
-    		Expect(text.String()).To(BeEmpty())
+    		Expect(ans).To(BeTrue())
+    		changeCheck := []string{"/spec/versions/name=v1/schema/openAPIV3Schema/properties/status/properties/nexus/required", "one required field added", "- testaddrequire"}
+    		changeCheckNot := []string{"/spec/versions/name=v1/schema/openAPIV3Schema/properties/status/properties/nexus/properties/addedField"}
+    		for _, v := range changeCheck {
+    			Expect(text.String()).Should(ContainSubstring(v))
+    		}
+    		for _, v := range changeCheckNot {
+    			Expect(text.String()).ShouldNot(ContainSubstring(v))
+    		}
     	})
     	It("should report change in nexus annotation", func() {
     		ans, text, err := CompareFiles([]byte(baseSpec), []byte(changeAnnotation))
@@ Expand All / @@ -59,6 +66,15 @@ var _ = Describe("Compare lib tests", func() { @@
     			Expect(text.String()).Should(ContainSubstring(v))
     		}
     	})
+    	It("it shouldn't filter out added required field", func() {
+    		ans, text, err := CompareFiles([]byte(removeRequired), []byte(baseSpec))
+    		Expect(err).ToNot(HaveOccurred())
+    		Expect(ans).To(BeTrue())
+    		changeCheck := []string{"spec changes", "/spec/versions/name=v1/schema/openAPIV3Schema/properties/spec", "one field added", "required", "name"}
+    		for _, v := range changeCheck {
+    			Expect(text.String()).Should(ContainSubstring(v))
+    		}
+    	})
     })
     var baseSpec = `
@@ Expand Down Expand Up / @@ -228,6 +244,7 @@ spec: @@
                       required:
                         - sourceGeneration
                         - remoteGeneration
+                        - testaddrequire
                       type: object
                   type: object
               type: object
@@ Expand Down Expand Up / @@ -919,3 +936,43 @@ status: @@
       - v1
     `
+    var removeRequired = `
+    apiVersion: apiextensions.k8s.io/v1
+    kind: CustomResourceDefinition
+    metadata:
+      annotations:
+        nexus: |
+          {"name":"gns.IgnoreChild","hierarchy":["roots.root.tsm.tanzu.vmware.com","configs.config.tsm.tanzu.vmware.com","gnses.gns.tsm.tanzu.vmware.com"],"is_singleton":false,"nexus-rest-api-gen":{"uris":null}}
+      creationTimestamp: null
+      name: ignorechilds.gns.tsm.tanzu.vmware.com
+    spec:
+      conversion:
+        strategy: None
+      group: gns.tsm.tanzu.vmware.com
+      versions:
+        - name: v1
+          schema:
+            openAPIV3Schema:
+              properties:
+                spec:
+                  properties:
+                    name:
+                      type: string
+                  type: object
+                status:
+                  properties:
+                    nexus:
+                      properties:
+                        remoteGeneration:
+                          format: int64
+                          type: integer
+                        sourceGeneration:
+                          format: int64
+                          type: integer
+                      required:
+                        - sourceGeneration
+                        - remoteGeneration
+                      type: object
+                  type: object
+              type: object`

compiler/Dockerfile

-Original file line number
+Diff line change
@@ Expand Up / @@ -6,7 +6,7 @@ ADD cmd/nexus-openapi-gen /go/bin/nexus-openapi-gen @@
     ADD cmd/gqlgen /go/bin/gqlgen
     WORKDIR /go/src/github.com/vmware-tanzu/graph-framework-for-microservices/compiler
-    RUN mv .git .gitmodules kube-openapi gqlgen nexus ..
+    RUN mv .git .gitmodules kube-openapi gqlgen nexus install-validator common-library  ..
     RUN git config --global --add safe.directory '*' && git config --global --add url."https://github.com/".insteadOf "git@github.com:"
     RUN make init_submodules
@@ Expand All / @@ -23,4 +23,9 @@ RUN go mod download @@
     WORKDIR /go/src/github.com/vmware-tanzu/graph-framework-for-microservices/compiler
     RUN go mod download
+    WORKDIR /go/src/github.com/vmware-tanzu/graph-framework-for-microservices/install-validator
+    RUN CGO_ENABLED=0 go build --trimpath -o install.so ./cmd/validate
+    WORKDIR /go/src/github.com/vmware-tanzu/graph-framework-for-microservices/compiler
     CMD make docker.gitlab_credentials && make generate_code

compiler/Makefile

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -19,6 +19,7 @@ DATAMODEL_PATH ?= datamodel
  
    CONFIG_FILE ?= ""

    GENERATED_OUTPUT_DIRECTORY ?= generated

    COMPILER_SRC_DIRECTORY ?= ""

    FORCE ?= false

    NEXUS_KUBEOPENAPI_VERSION ?= 7416bd4754d3c0dd8b3fa37fff53d36594f11607

    NEXUS_GRAPHQLGEN_VERSION ?= 32f028bce22efeb70b47a640195bd969dbb337f0

    @@ -57,9 +58,10 @@ docker.builder:
  
    	docker build --no-cache -t ${BUILDER_NAME}:${BUILDER_TAG} builder/

    .PHONY: docker

    docker: init_submodules ${BUILDER_NAME}\:${BUILDER_TAG}.image.exists build_openapigen_in_container build_gqlgen_in_container

    docker: init_submodules ${BUILDER_NAME}\:${BUILDER_TAG}.image.exists build_openapigen_in_container build_gqlgen_in_container build_install_validator_in_container

    	git archive -o compiler.tar --format=tar HEAD

    	tar -rf compiler.tar ../.git ../.gitmodules ../gqlgen ../kube-openapi ../nexus

    	ls ..

    	tar -rf compiler.tar ../.git ../.gitmodules ../gqlgen ../kube-openapi ../nexus ../install-validator ../common-library

    	docker build --no-cache \

    		--build-arg BUILDER_TAG=${BUILDER_TAG} \

    		-t ${IMAGE_NAME}:${TAG} .

    @@ -97,6 +99,12 @@ build_gqlgen_in_container:
  
    build_gqlgen:

    	GOBIN=${PKG_NAME}/cmd go install ../gqlgen

    build_install_validator_in_container:

    	$(call run_in_container,make build_install_validator)

    build_install_validator:

    	cd ../install-validator && GOBIN=${PKG_NAME}/cmd go install ./cmd/validate

    .PHONY: unit-test

    unit-test:

    	ginkgo -cover ./pkg/...

    @@ -149,14 +157,15 @@ generate_code:
  
    	CRD_MODULE_PATH=${CRD_MODULE_PATH} go run cmd/nexus-sdk/main.go -config-file ${CONFIG_FILE} -dsl ${DATAMODEL_PATH} -crd-output _generated -log-level ${LOG_LEVEL}

    	mv _generated/api_names.sh scripts/

    	@echo "Nexus Compiler: Resolving datamodel dependencies"

    	cd _generated && ../scripts/pin_deps.sh ${COMPILER_SRC_DIRECTORY} && go mod tidy -e

    	cd _generated && ../scripts/pin_deps.sh ${COMPILER_SRC_DIRECTORY} && go mod tidy -e 2>/dev/null

    	@echo "Nexus Compiler: Generating kuberenetes APIs"

    	./scripts/generate_k8s_api.sh

    	@echo "Nexus Compiler: Generating openapi schema"

    	./scripts/generate_openapi_schema.sh

    	@echo "Nexus Compiler: Generating CRD yamls"

    	go run cmd/generate-openapischema/generate-openapischema.go -yamls-path _generated/crds

    	go run cmd/generate-openapischema/generate-openapischema.go -yamls-path _generated/crds -existing-CRDs-Path ${GENERATED_OUTPUT_DIRECTORY}/crds -force ${FORCE}

    	git checkout -- pkg/openapi_generator/openapi/openapi_generated.go

    	rm -rf ${GENERATED_OUTPUT_DIRECTORY}/{client,apis,crds,common,nexus-client,helper,nexus-gql}

    	cp -r _generated/{client,apis,crds,common,nexus-client,helper,nexus-gql} ${GENERATED_OUTPUT_DIRECTORY}

    	@echo "Nexus Compiler: Generating GRAPHQL PKG"

    	cd _generated && goimports -w .

    @@ -165,7 +174,7 @@ generate_code:
  
    	cd ${GOPATH}/src/nexustempmodule && cd nexus-gql && go mod init && \

    	go mod edit -replace nexustempmodule=${GOPATH}/src/nexustempmodule && \

    	${COMPILER_SRC_DIRECTORY}/scripts/pin_graphql_build_version.sh ${COMPILER_SRC_DIRECTORY} && \

    	go mod tidy -e && \

    	go mod tidy -e 2>/dev/null && \

    	CGO_ENABLED=1 GOOS=linux \

    	go build --trimpath -o graphql.so -buildmode=plugin server.go

    	@echo "Updating module name"

    @@ -176,6 +185,8 @@ generate_code:
  
    	@echo "Nexus Compiler: Moving files to output directory"

    	cp -r _generated/{client,apis,crds,common,nexus-client,helper,nexus-gql} ${GENERATED_OUTPUT_DIRECTORY}

    	cp -r ${GOPATH}/src/nexustempmodule/nexus-gql/graphql.so ${GENERATED_OUTPUT_DIRECTORY}/nexus-gql

    	mkdir -p ${GENERATED_OUTPUT_DIRECTORY}/install-validator

    	cp -r ${GOPATH}/src/github.com/vmware-tanzu/graph-framework-for-microservices/install-validator/install.so ${GENERATED_OUTPUT_DIRECTORY}/install-validator/install.so || echo "Could not find install.so file, skipping"

    	@echo "Nexus Compiler: Compiler code generation completed"

    .PHONY: test_generate_code_in_container

compiler/cmd/generate-openapischema/generate-openapischema.go

-Original file line number
+Diff line change
@@ Expand Up / @@ -3,6 +3,7 @@ package main @@
     import (
     	"flag"
     	"fmt"
+    	"strconv"
     	"strings"
     	generator "github.com/vmware-tanzu/graph-framework-for-microservices/compiler/pkg/openapi_generator"
@@ Expand All / @@ -11,13 +12,24 @@ import ( @@
     )
     func main() {
-    	var yamlsPath string
+    	var (
+    		yamlsPath        string
+    		existingCRDsPath string
+    		forceUpgrade     string // Used to denote the forced upgrade of a data model.
+    	)
     	flag.StringVar(&yamlsPath, "yamls-path", "", "Path to directory containing CRD YAML definitions")
+    	flag.StringVar(&existingCRDsPath, "existing-CRDs-Path", "", "Path to directory containing existing CRD YAML definitions")
+    	flag.StringVar(&forceUpgrade, "force", "", "Set to true to force the nexus datamodel upgrade. \"+\n\t\t\t\"Defaults to `false`")
     	flag.Parse()
     	if yamlsPath == "" {
     		panic("yamls-path is empty. Run with -h for help")
     	}
+    	force, err := strconv.ParseBool(forceUpgrade)
+    	if err != nil {
+    		panic(fmt.Sprintf("invalid flag for nexus datamodel upgrade %v", err))
+    	}
     	ref := func(pkg string) spec.Ref {
     		r, err := spec.NewRef(strings.ToLower(pkg))
     		if err != nil {
@@ Expand Down Expand Up / @@ -48,4 +60,8 @@ func main() { @@
     	if err != nil {
     		panic(err)
     	}
+    	if err = generator.CheckBackwardCompatibility(existingCRDsPath, yamlsPath, force); err != nil {
+    		panic(fmt.Sprintf("Error occurred when checking datamodel compatibility: %v", err))
+    	}
     }

compiler/go.mod

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -7,51 +7,69 @@ require (
  
    	github.com/ghodss/yaml v1.0.0

    	github.com/gogo/protobuf v1.3.2

    	github.com/onsi/ginkgo v1.16.5

    	github.com/onsi/gomega v1.19.0

    	github.com/onsi/gomega v1.20.2

    	github.com/sirupsen/logrus v1.8.1

    	github.com/vmware-tanzu/graph-framework-for-microservices/common-library v0.0.0-20221129104902-06818e531062

    	github.com/vmware-tanzu/graph-framework-for-microservices/gqlgen v0.0.0-00010101000000-000000000000

    	github.com/vmware-tanzu/graph-framework-for-microservices/kube-openapi v0.0.0-20220603123335-7416bd4754d3

    	github.com/vmware-tanzu/graph-framework-for-microservices/nexus v0.0.0-00010101000000-000000000000

    	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3

    	golang.org/x/text v0.3.7

    	golang.org/x/tools v0.1.10

    	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4

    	golang.org/x/text v0.3.8

    	golang.org/x/tools v0.1.12

    	k8s.io/apiextensions-apiserver v0.24.0

    	k8s.io/apimachinery v0.24.0

    	k8s.io/gengo v0.0.0-20220307231824-4627b89bbf1b

    )

    require (

    	github.com/BurntSushi/toml v1.2.1 // indirect

    	github.com/agnivade/levenshtein v1.1.1 // indirect

    	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect

    	github.com/elliotchance/orderedmap v1.4.0 // indirect

    	github.com/emicklei/go-restful v2.15.0+incompatible // indirect

    	github.com/fsnotify/fsnotify v1.5.1 // indirect

    	github.com/fsnotify/fsnotify v1.5.4 // indirect

    	github.com/go-logr/logr v1.2.3 // indirect

    	github.com/go-openapi/jsonpointer v0.19.5 // indirect

    	github.com/go-openapi/jsonreference v0.20.0 // indirect

    	github.com/go-openapi/swag v0.21.1 // indirect

    	github.com/golang/protobuf v1.5.2 // indirect

    	github.com/gonvenience/bunt v1.3.4 // indirect

    	github.com/gonvenience/neat v1.3.11 // indirect

    	github.com/gonvenience/term v1.0.2 // indirect

    	github.com/gonvenience/text v1.0.7 // indirect

    	github.com/gonvenience/wrap v1.1.2 // indirect

    	github.com/gonvenience/ytbx v1.4.4 // indirect

    	github.com/google/gnostic v0.6.9 // indirect

    	github.com/google/go-cmp v0.5.9 // indirect

    	github.com/google/gofuzz v1.2.0 // indirect

    	github.com/homeport/dyff v1.5.6 // indirect

    	github.com/josharian/intern v1.0.0 // indirect

    	github.com/json-iterator/go v1.1.12 // indirect

    	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect

    	github.com/mailru/easyjson v0.7.6 // indirect

    	github.com/mattn/go-ciede2000 v0.0.0-20170301095244-782e8c62fec3 // indirect

    	github.com/mattn/go-isatty v0.0.16 // indirect

    	github.com/mitchellh/go-ps v1.0.0 // indirect

    	github.com/mitchellh/hashstructure v1.1.0 // indirect

    	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect

    	github.com/modern-go/reflect2 v1.0.2 // indirect

    	github.com/nxadm/tail v1.4.8 // indirect

    	github.com/russross/blackfriday/v2 v2.1.0 // indirect

    	github.com/sergi/go-diff v1.2.0 // indirect

    	github.com/texttheater/golang-levenshtein v1.0.1 // indirect

    	github.com/urfave/cli/v2 v2.8.1 // indirect

    	github.com/vektah/gqlparser/v2 v2.5.0 // indirect

    	github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 // indirect

    	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect

    	golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect

    	golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect

    	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect

    	golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect

    	golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde // indirect

    	golang.org/x/sys v0.1.0 // indirect

    	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect

    	google.golang.org/protobuf v1.28.0 // indirect

    	gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect

    	gopkg.in/inf.v0 v0.9.1 // indirect

    	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect

    	gopkg.in/yaml.v2 v2.4.0 // indirect

    	gopkg.in/yaml.v3 v3.0.1 // indirect

    	k8s.io/apimachinery v0.24.0 // indirect

    	k8s.io/klog/v2 v2.60.1 // indirect

    	k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect

    	sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect

    @@ -63,3 +81,5 @@ replace github.com/vmware-tanzu/graph-framework-for-microservices/kube-openapi =
  
    replace github.com/vmware-tanzu/graph-framework-for-microservices/gqlgen => ../gqlgen

    replace github.com/vmware-tanzu/graph-framework-for-microservices/nexus => ../nexus

    replace github.com/vmware-tanzu/graph-framework-for-microservices/common-library => ../common-library

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

NPT-684 DM Upgrade Integration #107

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

NPT-684 DM Upgrade Integration #107

Are you sure you want to change the base?

Uh oh!

NPT-684 DM Upgrade Integration #107

Uh oh!

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!