feat: add support for StatefulSets (#1088)

* feat: add support for StatefulSets * fix: resolve CI errors * fix: resolve license check errors * fix: resolve issue with duplicate QA keys * fix: check if cluster supports statefulsets * fix: apply requested changes * fix: don't create ingress/route automatically Signed-off-by: Soumil Paranjpay <soumil.paranjpay@gmail.com>
konveyor · Sep 21, 2023 · 1862c9c · 1862c9c
1 parent 1c32feb
commit 1862c9c
Show file tree

Hide file tree

Showing 21 changed files with 142 additions and 28 deletions.
diff --git a/common/constants.go b/common/constants.go
@@ -89,6 +89,8 @@ const (
 	ConfigStoragesKey = BaseKey + d + "storages"
 	//ConfigMinReplicasKey represents Ingress host Key
 	ConfigMinReplicasKey = BaseKey + d + "minreplicas"
+	//ConfigStatefulSetKey represents whether the IR should generate a StatefulSet
+	ConfigStatefulSetKey = "statefulset"
 	//ConfigPortsForServiceKeySegment represents the ports used for service
 	ConfigPortsForServiceKeySegment = "ports"
 	//ConfigPortForServiceKeySegment represents the port used for service

diff --git a/transformer/compose/composegenerator.go b/transformer/compose/composegenerator.go
@@ -23,7 +23,9 @@ import (
 	composetypes "github.com/docker/cli/cli/compose/types"
 	"github.com/konveyor/move2kube/common"
 	"github.com/konveyor/move2kube/environment"
+	"github.com/konveyor/move2kube/transformer/kubernetes"
 	"github.com/konveyor/move2kube/transformer/kubernetes/irpreprocessor"
+	"github.com/konveyor/move2kube/types/collection"
 	irtypes "github.com/konveyor/move2kube/types/ir"
 	transformertypes "github.com/konveyor/move2kube/types/transformer"
 	"github.com/sirupsen/logrus"
@@ -93,8 +95,14 @@ func (t *ComposeGenerator) Transform(newArtifacts []transformertypes.Artifact, a
 			logrus.Errorf("unable to load config for Transformer into %T : %s", ir, err)
 			continue
 		}
+
+		var clusterConfig collection.ClusterMetadata
+		if err := a.GetConfig(kubernetes.ClusterMetadata, &clusterConfig); err != nil {
+			logrus.Debugf("failed to load config for Transformer into %T . Error: %q", clusterConfig, err)
+		}
+
 		ir.Name = a.Name
-		preprocessedIR, err := irpreprocessor.Preprocess(ir)
+		preprocessedIR, err := irpreprocessor.Preprocess(ir, clusterConfig)
 		if err != nil {
 			logrus.Errorf("Unable to prepreocess IR : %s", err)
 		} else {

diff --git a/transformer/kubernetes/apiresource/deployment.go b/transformer/kubernetes/apiresource/deployment.go
@@ -46,6 +46,8 @@ const (
 	replicationControllerKind string = "ReplicationController"
 	// daemonSetKind defines DaemonSet Kind
 	daemonSetKind string = "DaemonSet"
+	// statefulSetKind defines StatefulSet Kind
+	statefulSetKind string = "StatefulSet"
 )
 
 // Deployment handles all objects like a Deployment
@@ -54,7 +56,7 @@ type Deployment struct {
 
 // getSupportedKinds returns kinds supported by the deployment
 func (d *Deployment) getSupportedKinds() []string {
-	return []string{podKind, jobKind, common.DeploymentKind, deploymentConfigKind, replicationControllerKind, daemonSetKind}
+	return []string{podKind, jobKind, common.DeploymentKind, deploymentConfigKind, replicationControllerKind, daemonSetKind, statefulSetKind}
 }
 
 // createNewResources converts ir to runtime object
@@ -67,6 +69,11 @@ func (d *Deployment) createNewResources(ir irtypes.EnhancedIR, supportedKinds []
 				logrus.Errorf("Creating Daemonset even though not supported by target cluster.")
 			}
 			obj = d.createDaemonSet(service, targetCluster.Spec)
+		} else if service.StatefulSet {
+			if !common.IsPresent(supportedKinds, statefulSetKind) {
+				logrus.Errorf("Creating Statefulset even though not supported by target cluster.")
+			}
+			obj = d.createStatefulSet(service, targetCluster.Spec)
 		} else if service.RestartPolicy == core.RestartPolicyNever || service.RestartPolicy == core.RestartPolicyOnFailure {
 			if common.IsPresent(supportedKinds, jobKind) {
 				obj = d.createJob(service, targetCluster.Spec)
@@ -266,6 +273,36 @@ func (d *Deployment) createJob(service irtypes.Service, cluster collecttypes.Clu
 	return &pod
 }
 
+func (d *Deployment) createStatefulSet(service irtypes.Service, cluster collecttypes.ClusterMetadataSpec) *apps.StatefulSet {
+	podSpec := service.PodSpec
+	podSpec = irtypes.PodSpec(d.convertVolumesKindsByPolicy(core.PodSpec(podSpec), cluster))
+	podSpec.RestartPolicy = core.RestartPolicyAlways
+	meta := metav1.ObjectMeta{
+		Name:        service.Name,
+		Labels:      getPodLabels(service.Name, service.Networks),
+		Annotations: getAnnotations(service),
+	}
+	statefulset := apps.StatefulSet{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       statefulSetKind,
+			APIVersion: metav1.SchemeGroupVersion.String(),
+		},
+		ObjectMeta: meta,
+		Spec: apps.StatefulSetSpec{
+			Replicas: int32(service.Replicas),
+			Selector: &metav1.LabelSelector{
+				MatchLabels: getServiceLabels(meta.Name),
+			},
+			Template: core.PodTemplateSpec{
+				ObjectMeta: meta,
+				Spec:       core.PodSpec(podSpec),
+			},
+			ServiceName: service.Name,
+		},
+	}
+	return &statefulset
+}
+
 // Conversions section
 
 func (d *Deployment) toDeploymentConfig(meta metav1.ObjectMeta, podspec core.PodSpec, replicas int32, cluster collecttypes.ClusterMetadataSpec) *okdappsv1.DeploymentConfig {

diff --git a/transformer/kubernetes/apiresource/service.go b/transformer/kubernetes/apiresource/service.go
@@ -470,7 +470,7 @@ func (d *Service) createService(service irtypes.Service) *core.Service {
 			Ports:    ports,
 		},
 	}
-	if len(ports) == 0 {
+	if len(ports) == 0 || service.StatefulSet {
 		svc.Spec.ClusterIP = "None"
 	}
 	return svc

diff --git a/transformer/kubernetes/argocdtransformer.go b/transformer/kubernetes/argocdtransformer.go
@@ -100,7 +100,7 @@ func (t *ArgoCD) Transform(newArtifacts []transformertypes.Artifact, alreadySeen
 			continue
 		}
 		ir.Name = newArtifact.Name
-		preprocessedIR, err := irpreprocessor.Preprocess(ir)
+		preprocessedIR, err := irpreprocessor.Preprocess(ir, clusterConfig)
 		if err != nil {
 			logrus.Errorf("Unable to prepreocess IR : %s", err)
 		} else {

diff --git a/transformer/kubernetes/buildconfigtransformer.go b/transformer/kubernetes/buildconfigtransformer.go
@@ -110,7 +110,7 @@ func (t *BuildConfig) Transform(newArtifacts []transformertypes.Artifact, alread
 			continue
 		}
 		ir.Name = newArtifact.Name
-		preprocessedIR, err := irpreprocessor.Preprocess(ir)
+		preprocessedIR, err := irpreprocessor.Preprocess(ir, clusterConfig)
 		if err != nil {
 			logrus.Errorf("failed to prepreocess the IR. Error: %q", err)
 		} else {

diff --git a/transformer/kubernetes/irpreprocessor/imagepullpolicypreprocessor.go b/transformer/kubernetes/irpreprocessor/imagepullpolicypreprocessor.go
@@ -17,6 +17,7 @@
 package irpreprocessor
 
 import (
+	"github.com/konveyor/move2kube/types/collection"
 	irtypes "github.com/konveyor/move2kube/types/ir"
 	core "k8s.io/kubernetes/pkg/apis/core"
 )
@@ -25,7 +26,7 @@ import (
 type imagePullPolicyPreprocessor struct {
 }
 
-func (ep imagePullPolicyPreprocessor) preprocess(ir irtypes.IR) (irtypes.IR, error) {
+func (ep imagePullPolicyPreprocessor) preprocess(ir irtypes.IR, targetCluster collection.ClusterMetadata) (irtypes.IR, error) {
 	for k, scObj := range ir.Services {
 		for i := range scObj.Containers {
 			scObj.Containers[i].ImagePullPolicy = core.PullAlways

diff --git a/transformer/kubernetes/irpreprocessor/imagepullpolicypreprocessor_test.go b/transformer/kubernetes/irpreprocessor/imagepullpolicypreprocessor_test.go
@@ -20,13 +20,16 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
+	"github.com/konveyor/move2kube/types/collection"
 	irtypes "github.com/konveyor/move2kube/types/ir"
 	"github.com/sirupsen/logrus"
 	core "k8s.io/kubernetes/pkg/apis/core"
 )
 
 func TestImagePullPolicyOptimizer(t *testing.T) {
 	logrus.SetLevel(logrus.DebugLevel)
+	var clusterConfig collection.ClusterMetadata
+
 
 	t.Run("IR with no services", func(t *testing.T) {
 		// Setup
@@ -35,7 +38,7 @@ func TestImagePullPolicyOptimizer(t *testing.T) {
 		want := getIRWithoutServices()
 
 		// Test
-		actual, err := imagePullPolicyPreprocessor.preprocess(ir)
+		actual, err := imagePullPolicyPreprocessor.preprocess(ir, clusterConfig)
 		if err != nil {
 			t.Fatal("Failed to get the expected. Error:", err)
 		}
@@ -51,7 +54,7 @@ func TestImagePullPolicyOptimizer(t *testing.T) {
 		want := getIRWithServicesAndWithoutContainers()
 
 		// Test
-		actual, err := imagePullPolicyPreprocessor.preprocess(ir)
+		actual, err := imagePullPolicyPreprocessor.preprocess(ir, clusterConfig)
 		if err != nil {
 			t.Fatal("Failed to get the expected. Error:", err)
 		}
@@ -83,7 +86,7 @@ func TestImagePullPolicyOptimizer(t *testing.T) {
 		want := getIRWithImagePullPolicySetAsAlways()
 
 		// Test
-		actual, err := imagePullPolicyPreprocessor.preprocess(ir)
+		actual, err := imagePullPolicyPreprocessor.preprocess(ir, clusterConfig)
 		if err != nil {
 			t.Fatal("Failed to get the expected. Error:", err)
 		}
@@ -99,7 +102,7 @@ func TestImagePullPolicyOptimizer(t *testing.T) {
 		want := getIRWithImagePullPolicySetAsAlways()
 
 		// Test
-		actual, err := imagePullPolicyPreprocessor.preprocess(ir)
+		actual, err := imagePullPolicyPreprocessor.preprocess(ir, clusterConfig)
 		if err != nil {
 			t.Fatal("Failed to get the expected. Error:", err)
 		}

diff --git a/transformer/kubernetes/irpreprocessor/ingresspreprocessor.go b/transformer/kubernetes/irpreprocessor/ingresspreprocessor.go
@@ -22,6 +22,7 @@ import (
 
 	"github.com/konveyor/move2kube/common"
 	"github.com/konveyor/move2kube/qaengine"
+	"github.com/konveyor/move2kube/types/collection"
 	irtypes "github.com/konveyor/move2kube/types/ir"
 	"github.com/spf13/cast"
 	core "k8s.io/kubernetes/pkg/apis/core"
@@ -31,7 +32,7 @@ import (
 type ingressPreprocessor struct {
 }
 
-func (opt *ingressPreprocessor) preprocess(ir irtypes.IR) (irtypes.IR, error) {
+func (opt *ingressPreprocessor) preprocess(ir irtypes.IR, targetCluster collection.ClusterMetadata) (irtypes.IR, error) {
 	for serviceName, service := range ir.Services {
 		tempService := ir.Services[serviceName]
 		for portForwardingIdx, portForwarding := range service.ServiceToPodPortForwardings {
@@ -41,6 +42,10 @@ func (opt *ingressPreprocessor) preprocess(ir irtypes.IR) (irtypes.IR, error) {
 			if portForwarding.ServiceRelPath == "" {
 				portForwarding.ServiceRelPath = "/" + serviceName
 			}
+			// Create Headless Services for services that are to be converted into StatefulSets
+			if service.StatefulSet {
+				portForwarding.ServiceType = core.ServiceTypeClusterIP
+			}
 			noneServiceType := "Don't create service"
 			portKeyPart := common.JoinQASubKeys(common.ConfigServicesKey, `"`+serviceName+`"`, `"`+cast.ToString(portForwarding.ServicePort.Number)+`"`)
 			options := []string{common.IngressKind, string(core.ServiceTypeLoadBalancer), string(core.ServiceTypeNodePort), string(core.ServiceTypeClusterIP), noneServiceType}

diff --git a/transformer/kubernetes/irpreprocessor/irpreprocessor.go b/transformer/kubernetes/irpreprocessor/irpreprocessor.go
@@ -17,29 +17,31 @@
 package irpreprocessor
 
 import (
+	"github.com/konveyor/move2kube/types/collection"
 	irtypes "github.com/konveyor/move2kube/types/ir"
 	"github.com/sirupsen/logrus"
 )
 
 // irpreprocessor optimizes the configuration
 type irpreprocessor interface {
-	preprocess(sourceir irtypes.IR) (irtypes.IR, error)
+	preprocess(sourceir irtypes.IR, targetCluster collection.ClusterMetadata) (irtypes.IR, error)
 }
 
 // getIRPreprocessors returns optimizers
 func getIRPreprocessors() []irpreprocessor {
-	var l = []irpreprocessor{new(mergePreprocessor), new(normalizeCharacterPreprocessor), new(ingressPreprocessor), new(replicaPreprocessor), new(imagePullPolicyPreprocessor), new(registryPreProcessor)}
+	var l = []irpreprocessor{new(mergePreprocessor), new(normalizeCharacterPreprocessor), new(statefulsetPreprocessor), new(ingressPreprocessor), new(replicaPreprocessor), 
+		new(imagePullPolicyPreprocessor), new(registryPreProcessor)}
 	return l
 }
 
 // Preprocess preprocesses IR before application artifacts are generated
-func Preprocess(ir irtypes.IR) (irtypes.IR, error) {
+func Preprocess(ir irtypes.IR, targetCluster collection.ClusterMetadata) (irtypes.IR, error) {
 	optimizers := getIRPreprocessors()
 	logrus.Debug("Begin Optimization")
 	for _, o := range optimizers {
 		logrus.Debugf("[%T] Begin Optimization", o)
 		var err error
-		ir, err = o.preprocess(ir)
+		ir, err = o.preprocess(ir, targetCluster)
 		if err != nil {
 			logrus.Warnf("[%T] Failed : %s", o, err.Error())
 		} else {

diff --git a/transformer/kubernetes/irpreprocessor/mergepreprocessor.go b/transformer/kubernetes/irpreprocessor/mergepreprocessor.go
@@ -17,6 +17,7 @@
 package irpreprocessor
 
 import (
+	"github.com/konveyor/move2kube/types/collection"
 	irtypes "github.com/konveyor/move2kube/types/ir"
 	core "k8s.io/kubernetes/pkg/apis/core"
 	"k8s.io/kubernetes/pkg/apis/networking"
@@ -27,7 +28,7 @@ type mergePreprocessor struct {
 }
 
 // Preprocesses the port forwardings
-func (opt *mergePreprocessor) preprocess(ir irtypes.IR) (irtypes.IR, error) {
+func (opt *mergePreprocessor) preprocess(ir irtypes.IR, targetCluster collection.ClusterMetadata) (irtypes.IR, error) {
 	for serviceName, service := range ir.Services {
 		service.Containers = opt.mergeContainers(service.Containers)
 		pfs := service.ServiceToPodPortForwardings

diff --git a/transformer/kubernetes/irpreprocessor/normalizecharacterspreprocessor.go b/transformer/kubernetes/irpreprocessor/normalizecharacterspreprocessor.go
@@ -20,6 +20,7 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/konveyor/move2kube/types/collection"
 	irtypes "github.com/konveyor/move2kube/types/ir"
 	core "k8s.io/kubernetes/pkg/apis/core"
 )
@@ -28,7 +29,7 @@ import (
 type normalizeCharacterPreprocessor struct {
 }
 
-func (po normalizeCharacterPreprocessor) preprocess(ir irtypes.IR) (irtypes.IR, error) {
+func (po normalizeCharacterPreprocessor) preprocess(ir irtypes.IR, targetCluster collection.ClusterMetadata) (irtypes.IR, error) {
 	//TODO: Make this generic to ensure all fields have valid names
 	for i := range ir.Services {
 		for j := range ir.Services[i].Containers {

diff --git a/transformer/kubernetes/irpreprocessor/normalizecharacterspreprocessor_test.go b/transformer/kubernetes/irpreprocessor/normalizecharacterspreprocessor_test.go
@@ -20,6 +20,7 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
+	"github.com/konveyor/move2kube/types/collection"
 	irtypes "github.com/konveyor/move2kube/types/ir"
 	"github.com/sirupsen/logrus"
 	core "k8s.io/kubernetes/pkg/apis/core"
@@ -67,6 +68,7 @@ func TestStripQuotation(t *testing.T) {
 
 func TestOptimize(t *testing.T) {
 	logrus.SetLevel(logrus.DebugLevel)
+	var clusterConfig collection.ClusterMetadata
 
 	t.Run("IR with no services", func(t *testing.T) {
 		// Setup
@@ -75,7 +77,7 @@ func TestOptimize(t *testing.T) {
 		want := getIRWithoutServices()
 
 		// Test
-		actual, err := normalizeCharacterPreprocessor.preprocess(ir)
+		actual, err := normalizeCharacterPreprocessor.preprocess(ir, clusterConfig)
 		if err != nil {
 			t.Fatal("Failed to get the expected. Error:", err)
 		}
@@ -91,7 +93,7 @@ func TestOptimize(t *testing.T) {
 		want := getIRWithServicesAndWithoutContainers()
 
 		// Test
-		actual, err := normalizeCharacterPreprocessor.preprocess(ir)
+		actual, err := normalizeCharacterPreprocessor.preprocess(ir, clusterConfig)
 		if err != nil {
 			t.Fatal("Failed to get the expected. Error:", err)
 		}
@@ -107,7 +109,7 @@ func TestOptimize(t *testing.T) {
 		want := getIRWithServicesAndContainersWithoutEnv()
 
 		// Test
-		actual, err := normalizeCharacterPreprocessor.preprocess(ir)
+		actual, err := normalizeCharacterPreprocessor.preprocess(ir, clusterConfig)
 		if err != nil {
 			t.Fatal("Failed to get the expected. Error:", err)
 		}
@@ -123,7 +125,7 @@ func TestOptimize(t *testing.T) {
 		want := getIRWithServicesAndContainersWithValidEnv()
 
 		// Test
-		actual, err := normalizeCharacterPreprocessor.preprocess(ir)
+		actual, err := normalizeCharacterPreprocessor.preprocess(ir, clusterConfig)
 		if err != nil {
 			t.Fatal("Failed to get the expected. Error:", err)
 		}
@@ -163,7 +165,7 @@ func TestOptimize(t *testing.T) {
 		want := getExpectedIR()
 
 		// Test
-		actual, err := normalizeCharacterPreprocessor.preprocess(ir)
+		actual, err := normalizeCharacterPreprocessor.preprocess(ir, clusterConfig)
 		if err != nil {
 			t.Fatal("Failed to get the expected. Error:", err)
 		}
@@ -202,7 +204,7 @@ func TestOptimize(t *testing.T) {
 		want := getExpectedIRWithAffinityInContainer()
 
 		// Test
-		actual, err := normalizeCharacterPreprocessor.preprocess(ir)
+		actual, err := normalizeCharacterPreprocessor.preprocess(ir, clusterConfig)
 		if err != nil {
 			t.Fatal("Failed to get the expected. Error:", err)
 		}

diff --git a/transformer/kubernetes/irpreprocessor/registrypreprocessor.go b/transformer/kubernetes/irpreprocessor/registrypreprocessor.go
@@ -27,6 +27,7 @@ import (
 	dockerclitypes "github.com/docker/cli/cli/config/types"
 	"github.com/konveyor/move2kube/common"
 	"github.com/konveyor/move2kube/qaengine"
+	"github.com/konveyor/move2kube/types/collection"
 	irtypes "github.com/konveyor/move2kube/types/ir"
 	"github.com/konveyor/move2kube/types/qaengine/commonqa"
 	"github.com/sirupsen/logrus"
@@ -51,7 +52,7 @@ const (
 	imagePullSecretSuffix = "-imagepullsecret"
 )
 
-func (p registryPreProcessor) preprocess(ir irtypes.IR) (irtypes.IR, error) {
+func (p registryPreProcessor) preprocess(ir irtypes.IR, targetCluster collection.ClusterMetadata) (irtypes.IR, error) {
 	// find all the new images that we are going to create
 
 	newImageNames := []string{}