ingress_utils.go 41 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159
  1. /*
  2. Copyright 2015 The Kubernetes Authors.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. */
  13. package ingress
  14. import (
  15. "bytes"
  16. "context"
  17. "crypto/rand"
  18. "crypto/rsa"
  19. "crypto/tls"
  20. "crypto/x509"
  21. "crypto/x509/pkix"
  22. "encoding/pem"
  23. "fmt"
  24. "io/ioutil"
  25. "math/big"
  26. "net"
  27. "net/http"
  28. "path/filepath"
  29. "regexp"
  30. "strconv"
  31. "strings"
  32. "time"
  33. compute "google.golang.org/api/compute/v1"
  34. "k8s.io/klog"
  35. appsv1 "k8s.io/api/apps/v1"
  36. v1 "k8s.io/api/core/v1"
  37. networkingv1beta1 "k8s.io/api/networking/v1beta1"
  38. apierrors "k8s.io/apimachinery/pkg/api/errors"
  39. metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
  40. "k8s.io/apimachinery/pkg/fields"
  41. "k8s.io/apimachinery/pkg/labels"
  42. "k8s.io/apimachinery/pkg/runtime"
  43. "k8s.io/apimachinery/pkg/runtime/schema"
  44. "k8s.io/apimachinery/pkg/util/intstr"
  45. utilnet "k8s.io/apimachinery/pkg/util/net"
  46. "k8s.io/apimachinery/pkg/util/sets"
  47. "k8s.io/apimachinery/pkg/util/wait"
  48. utilyaml "k8s.io/apimachinery/pkg/util/yaml"
  49. clientset "k8s.io/client-go/kubernetes"
  50. "k8s.io/client-go/kubernetes/scheme"
  51. "k8s.io/kubernetes/test/e2e/framework"
  52. e2eservice "k8s.io/kubernetes/test/e2e/framework/service"
  53. "k8s.io/kubernetes/test/e2e/framework/testfiles"
  54. testutils "k8s.io/kubernetes/test/utils"
  55. imageutils "k8s.io/kubernetes/test/utils/image"
  56. "github.com/onsi/ginkgo"
  57. )
  58. const (
  59. rsaBits = 2048
  60. validFor = 365 * 24 * time.Hour
  61. // IngressClassKey is ingress class annotation defined in ingress repository.
  62. // TODO: All these annotations should be reused from
  63. // ingress-gce/pkg/annotations instead of duplicating them here.
  64. IngressClassKey = "kubernetes.io/ingress.class"
  65. // MulticlusterIngressClassValue is ingress class annotation value for multi cluster ingress.
  66. MulticlusterIngressClassValue = "gce-multi-cluster"
  67. // IngressStaticIPKey is static IP annotation defined in ingress repository.
  68. IngressStaticIPKey = "kubernetes.io/ingress.global-static-ip-name"
  69. // IngressAllowHTTPKey is Allow HTTP annotation defined in ingress repository.
  70. IngressAllowHTTPKey = "kubernetes.io/ingress.allow-http"
  71. // IngressPreSharedCertKey is Pre-shared-cert annotation defined in ingress repository.
  72. IngressPreSharedCertKey = "ingress.gcp.kubernetes.io/pre-shared-cert"
  73. // ServiceApplicationProtocolKey annotation defined in ingress repository.
  74. ServiceApplicationProtocolKey = "service.alpha.kubernetes.io/app-protocols"
  75. // Name of the default http backend service
  76. defaultBackendName = "default-http-backend"
  77. // IngressManifestPath is the parent path to yaml test manifests.
  78. IngressManifestPath = "test/e2e/testing-manifests/ingress"
  79. // GCEIngressManifestPath is the parent path to GCE-specific yaml test manifests.
  80. GCEIngressManifestPath = IngressManifestPath + "/gce"
  81. // IngressReqTimeout is the timeout on a single http request.
  82. IngressReqTimeout = 10 * time.Second
  83. // NEGAnnotation is NEG annotation.
  84. NEGAnnotation = "cloud.google.com/neg"
  85. // NEGStatusAnnotation is NEG status annotation.
  86. NEGStatusAnnotation = "cloud.google.com/neg-status"
  87. // StatusPrefix is prefix for annotation keys used by the ingress controller to specify the
  88. // names of GCP resources such as forwarding rules, url maps, target proxies, etc
  89. // that it created for the corresponding ingress.
  90. StatusPrefix = "ingress.kubernetes.io"
  91. // poll is how often to Poll pods, nodes and claims.
  92. poll = 2 * time.Second
  93. )
  94. // TestLogger is an interface for log.
  95. type TestLogger interface {
  96. Infof(format string, args ...interface{})
  97. Errorf(format string, args ...interface{})
  98. }
  99. // GLogger is test logger.
  100. type GLogger struct{}
  101. // Infof outputs log with info level.
  102. func (l *GLogger) Infof(format string, args ...interface{}) {
  103. klog.Infof(format, args...)
  104. }
  105. // Errorf outputs log with error level.
  106. func (l *GLogger) Errorf(format string, args ...interface{}) {
  107. klog.Errorf(format, args...)
  108. }
  109. // E2ELogger is test logger.
  110. type E2ELogger struct{}
  111. // Infof outputs log.
  112. func (l *E2ELogger) Infof(format string, args ...interface{}) {
  113. framework.Logf(format, args...)
  114. }
  115. // Errorf outputs log.
  116. func (l *E2ELogger) Errorf(format string, args ...interface{}) {
  117. framework.Logf(format, args...)
  118. }
  119. // ConformanceTests contains a closure with an entry and exit log line.
  120. type ConformanceTests struct {
  121. EntryLog string
  122. Execute func()
  123. ExitLog string
  124. }
  125. // NegStatus contains name and zone of the Network Endpoint Group
  126. // resources associated with this service.
  127. // Needs to be consistent with the NEG internal structs in ingress-gce.
  128. type NegStatus struct {
  129. // NetworkEndpointGroups returns the mapping between service port and NEG
  130. // resource. key is service port, value is the name of the NEG resource.
  131. NetworkEndpointGroups map[int32]string `json:"network_endpoint_groups,omitempty"`
  132. Zones []string `json:"zones,omitempty"`
  133. }
  134. // SimpleGET executes a get on the given url, returns error if non-200 returned.
  135. func SimpleGET(c *http.Client, url, host string) (string, error) {
  136. req, err := http.NewRequest("GET", url, nil)
  137. if err != nil {
  138. return "", err
  139. }
  140. req.Host = host
  141. res, err := c.Do(req)
  142. if err != nil {
  143. return "", err
  144. }
  145. defer res.Body.Close()
  146. rawBody, err := ioutil.ReadAll(res.Body)
  147. if err != nil {
  148. return "", err
  149. }
  150. body := string(rawBody)
  151. if res.StatusCode != http.StatusOK {
  152. err = fmt.Errorf(
  153. "GET returned http error %v", res.StatusCode)
  154. }
  155. return body, err
  156. }
  157. // PollURL polls till the url responds with a healthy http code. If
  158. // expectUnreachable is true, it breaks on first non-healthy http code instead.
  159. func PollURL(route, host string, timeout time.Duration, interval time.Duration, httpClient *http.Client, expectUnreachable bool) error {
  160. var lastBody string
  161. pollErr := wait.PollImmediate(interval, timeout, func() (bool, error) {
  162. var err error
  163. lastBody, err = SimpleGET(httpClient, route, host)
  164. if err != nil {
  165. framework.Logf("host %v path %v: %v unreachable", host, route, err)
  166. return expectUnreachable, nil
  167. }
  168. framework.Logf("host %v path %v: reached", host, route)
  169. return !expectUnreachable, nil
  170. })
  171. if pollErr != nil {
  172. return fmt.Errorf("Failed to execute a successful GET within %v, Last response body for %v, host %v:\n%v\n\n%v",
  173. timeout, route, host, lastBody, pollErr)
  174. }
  175. return nil
  176. }
  177. // CreateIngressComformanceTests generates an slice of sequential test cases:
  178. // a simple http ingress, ingress with HTTPS, ingress HTTPS with a modified hostname,
  179. // ingress https with a modified URLMap
  180. func CreateIngressComformanceTests(jig *TestJig, ns string, annotations map[string]string) []ConformanceTests {
  181. manifestPath := filepath.Join(IngressManifestPath, "http")
  182. // These constants match the manifests used in IngressManifestPath
  183. tlsHost := "foo.bar.com"
  184. tlsSecretName := "foo"
  185. updatedTLSHost := "foobar.com"
  186. updateURLMapHost := "bar.baz.com"
  187. updateURLMapPath := "/testurl"
  188. // Platform agnostic list of tests that must be satisfied by all controllers
  189. tests := []ConformanceTests{
  190. {
  191. fmt.Sprintf("should create a basic HTTP ingress"),
  192. func() { jig.CreateIngress(manifestPath, ns, annotations, annotations) },
  193. fmt.Sprintf("waiting for urls on basic HTTP ingress"),
  194. },
  195. {
  196. fmt.Sprintf("should terminate TLS for host %v", tlsHost),
  197. func() { jig.SetHTTPS(tlsSecretName, tlsHost) },
  198. fmt.Sprintf("waiting for HTTPS updates to reflect in ingress"),
  199. },
  200. {
  201. fmt.Sprintf("should update url map for host %v to expose a single url: %v", updateURLMapHost, updateURLMapPath),
  202. func() {
  203. var pathToFail string
  204. jig.Update(func(ing *networkingv1beta1.Ingress) {
  205. newRules := []networkingv1beta1.IngressRule{}
  206. for _, rule := range ing.Spec.Rules {
  207. if rule.Host != updateURLMapHost {
  208. newRules = append(newRules, rule)
  209. continue
  210. }
  211. existingPath := rule.IngressRuleValue.HTTP.Paths[0]
  212. pathToFail = existingPath.Path
  213. newRules = append(newRules, networkingv1beta1.IngressRule{
  214. Host: updateURLMapHost,
  215. IngressRuleValue: networkingv1beta1.IngressRuleValue{
  216. HTTP: &networkingv1beta1.HTTPIngressRuleValue{
  217. Paths: []networkingv1beta1.HTTPIngressPath{
  218. {
  219. Path: updateURLMapPath,
  220. Backend: existingPath.Backend,
  221. },
  222. },
  223. },
  224. },
  225. })
  226. }
  227. ing.Spec.Rules = newRules
  228. })
  229. ginkgo.By("Checking that " + pathToFail + " is not exposed by polling for failure")
  230. route := fmt.Sprintf("http://%v%v", jig.Address, pathToFail)
  231. framework.ExpectNoError(PollURL(route, updateURLMapHost, e2eservice.LoadBalancerCleanupTimeout, jig.PollInterval, &http.Client{Timeout: IngressReqTimeout}, true))
  232. },
  233. fmt.Sprintf("Waiting for path updates to reflect in L7"),
  234. },
  235. }
  236. // Skip the Update TLS cert test for kubemci: https://github.com/GoogleCloudPlatform/k8s-multicluster-ingress/issues/141.
  237. if jig.Class != MulticlusterIngressClassValue {
  238. tests = append(tests, ConformanceTests{
  239. fmt.Sprintf("should update SSL certificate with modified hostname %v", updatedTLSHost),
  240. func() {
  241. jig.Update(func(ing *networkingv1beta1.Ingress) {
  242. newRules := []networkingv1beta1.IngressRule{}
  243. for _, rule := range ing.Spec.Rules {
  244. if rule.Host != tlsHost {
  245. newRules = append(newRules, rule)
  246. continue
  247. }
  248. newRules = append(newRules, networkingv1beta1.IngressRule{
  249. Host: updatedTLSHost,
  250. IngressRuleValue: rule.IngressRuleValue,
  251. })
  252. }
  253. ing.Spec.Rules = newRules
  254. })
  255. jig.SetHTTPS(tlsSecretName, updatedTLSHost)
  256. },
  257. fmt.Sprintf("Waiting for updated certificates to accept requests for host %v", updatedTLSHost),
  258. })
  259. }
  260. return tests
  261. }
  262. // GenerateRSACerts generates a basic self signed certificate using a key length
  263. // of rsaBits, valid for validFor time.
  264. func GenerateRSACerts(host string, isCA bool) ([]byte, []byte, error) {
  265. if len(host) == 0 {
  266. return nil, nil, fmt.Errorf("Require a non-empty host for client hello")
  267. }
  268. priv, err := rsa.GenerateKey(rand.Reader, rsaBits)
  269. if err != nil {
  270. return nil, nil, fmt.Errorf("Failed to generate key: %v", err)
  271. }
  272. notBefore := time.Now()
  273. notAfter := notBefore.Add(validFor)
  274. serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
  275. serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
  276. if err != nil {
  277. return nil, nil, fmt.Errorf("failed to generate serial number: %s", err)
  278. }
  279. template := x509.Certificate{
  280. SerialNumber: serialNumber,
  281. Subject: pkix.Name{
  282. CommonName: "default",
  283. Organization: []string{"Acme Co"},
  284. },
  285. NotBefore: notBefore,
  286. NotAfter: notAfter,
  287. KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
  288. ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
  289. BasicConstraintsValid: true,
  290. }
  291. hosts := strings.Split(host, ",")
  292. for _, h := range hosts {
  293. if ip := net.ParseIP(h); ip != nil {
  294. template.IPAddresses = append(template.IPAddresses, ip)
  295. } else {
  296. template.DNSNames = append(template.DNSNames, h)
  297. }
  298. }
  299. if isCA {
  300. template.IsCA = true
  301. template.KeyUsage |= x509.KeyUsageCertSign
  302. }
  303. var keyOut, certOut bytes.Buffer
  304. derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
  305. if err != nil {
  306. return nil, nil, fmt.Errorf("Failed to create certificate: %s", err)
  307. }
  308. if err := pem.Encode(&certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
  309. return nil, nil, fmt.Errorf("Failed creating cert: %v", err)
  310. }
  311. if err := pem.Encode(&keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil {
  312. return nil, nil, fmt.Errorf("Failed creating key: %v", err)
  313. }
  314. return certOut.Bytes(), keyOut.Bytes(), nil
  315. }
  316. // buildTransportWithCA creates a transport for use in executing HTTPS requests with
  317. // the given certs. Note that the given rootCA must be configured with isCA=true.
  318. func buildTransportWithCA(serverName string, rootCA []byte) (*http.Transport, error) {
  319. pool := x509.NewCertPool()
  320. ok := pool.AppendCertsFromPEM(rootCA)
  321. if !ok {
  322. return nil, fmt.Errorf("Unable to load serverCA")
  323. }
  324. return utilnet.SetTransportDefaults(&http.Transport{
  325. TLSClientConfig: &tls.Config{
  326. InsecureSkipVerify: false,
  327. ServerName: serverName,
  328. RootCAs: pool,
  329. },
  330. }), nil
  331. }
  332. // BuildInsecureClient returns an insecure http client. Can be used for "curl -k".
  333. func BuildInsecureClient(timeout time.Duration) *http.Client {
  334. t := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
  335. return &http.Client{Timeout: timeout, Transport: utilnet.SetTransportDefaults(t)}
  336. }
  337. // createTLSSecret creates a secret containing TLS certificates.
  338. // If a secret with the same name already pathExists in the namespace of the
  339. // Ingress, it's updated.
  340. func createTLSSecret(kubeClient clientset.Interface, namespace, secretName string, hosts ...string) (host string, rootCA, privKey []byte, err error) {
  341. host = strings.Join(hosts, ",")
  342. framework.Logf("Generating RSA cert for host %v", host)
  343. cert, key, err := GenerateRSACerts(host, true)
  344. if err != nil {
  345. return
  346. }
  347. secret := &v1.Secret{
  348. ObjectMeta: metav1.ObjectMeta{
  349. Name: secretName,
  350. },
  351. Data: map[string][]byte{
  352. v1.TLSCertKey: cert,
  353. v1.TLSPrivateKeyKey: key,
  354. },
  355. }
  356. var s *v1.Secret
  357. if s, err = kubeClient.CoreV1().Secrets(namespace).Get(context.TODO(), secretName, metav1.GetOptions{}); err == nil {
  358. framework.Logf("Updating secret %v in ns %v with hosts %v", secret.Name, namespace, host)
  359. s.Data = secret.Data
  360. _, err = kubeClient.CoreV1().Secrets(namespace).Update(context.TODO(), s, metav1.UpdateOptions{})
  361. } else {
  362. framework.Logf("Creating secret %v in ns %v with hosts %v", secret.Name, namespace, host)
  363. _, err = kubeClient.CoreV1().Secrets(namespace).Create(context.TODO(), secret, metav1.CreateOptions{})
  364. }
  365. return host, cert, key, err
  366. }
  367. // TestJig holds the relevant state and parameters of the ingress test.
  368. type TestJig struct {
  369. Client clientset.Interface
  370. Logger TestLogger
  371. RootCAs map[string][]byte
  372. Address string
  373. Ingress *networkingv1beta1.Ingress
  374. // class is the value of the annotation keyed under
  375. // `kubernetes.io/ingress.class`. It's added to all ingresses created by
  376. // this jig.
  377. Class string
  378. // The interval used to poll urls
  379. PollInterval time.Duration
  380. }
  381. // NewIngressTestJig instantiates struct with client
  382. func NewIngressTestJig(c clientset.Interface) *TestJig {
  383. return &TestJig{
  384. Client: c,
  385. RootCAs: map[string][]byte{},
  386. PollInterval: e2eservice.LoadBalancerPollInterval,
  387. Logger: &E2ELogger{},
  388. }
  389. }
  390. // CreateIngress creates the Ingress and associated service/rc.
  391. // Required: ing.yaml, rc.yaml, svc.yaml must exist in manifestPath
  392. // Optional: secret.yaml, ingAnnotations
  393. // If ingAnnotations is specified it will overwrite any annotations in ing.yaml
  394. // If svcAnnotations is specified it will overwrite any annotations in svc.yaml
  395. func (j *TestJig) CreateIngress(manifestPath, ns string, ingAnnotations map[string]string, svcAnnotations map[string]string) {
  396. var err error
  397. read := func(file string) string {
  398. return string(testfiles.ReadOrDie(filepath.Join(manifestPath, file)))
  399. }
  400. exists := func(file string) bool {
  401. return testfiles.Exists(filepath.Join(manifestPath, file))
  402. }
  403. j.Logger.Infof("creating replication controller")
  404. framework.RunKubectlOrDieInput(ns, read("rc.yaml"), "create", "-f", "-", fmt.Sprintf("--namespace=%v", ns))
  405. j.Logger.Infof("creating service")
  406. framework.RunKubectlOrDieInput(ns, read("svc.yaml"), "create", "-f", "-", fmt.Sprintf("--namespace=%v", ns))
  407. if len(svcAnnotations) > 0 {
  408. svcList, err := j.Client.CoreV1().Services(ns).List(context.TODO(), metav1.ListOptions{})
  409. framework.ExpectNoError(err)
  410. for _, svc := range svcList.Items {
  411. svc.Annotations = svcAnnotations
  412. _, err = j.Client.CoreV1().Services(ns).Update(context.TODO(), &svc, metav1.UpdateOptions{})
  413. framework.ExpectNoError(err)
  414. }
  415. }
  416. if exists("secret.yaml") {
  417. j.Logger.Infof("creating secret")
  418. framework.RunKubectlOrDieInput(ns, read("secret.yaml"), "create", "-f", "-", fmt.Sprintf("--namespace=%v", ns))
  419. }
  420. j.Logger.Infof("Parsing ingress from %v", filepath.Join(manifestPath, "ing.yaml"))
  421. j.Ingress, err = ingressFromManifest(filepath.Join(manifestPath, "ing.yaml"))
  422. framework.ExpectNoError(err)
  423. j.Ingress.Namespace = ns
  424. j.Ingress.Annotations = map[string]string{IngressClassKey: j.Class}
  425. for k, v := range ingAnnotations {
  426. j.Ingress.Annotations[k] = v
  427. }
  428. j.Logger.Infof(fmt.Sprintf("creating " + j.Ingress.Name + " ingress"))
  429. j.Ingress, err = j.runCreate(j.Ingress)
  430. framework.ExpectNoError(err)
  431. }
  432. // marshalToYaml marshals an object into YAML for a given GroupVersion.
  433. // The object must be known in SupportedMediaTypes() for the Codecs under "client-go/kubernetes/scheme".
  434. func marshalToYaml(obj runtime.Object, gv schema.GroupVersion) ([]byte, error) {
  435. mediaType := "application/yaml"
  436. info, ok := runtime.SerializerInfoForMediaType(scheme.Codecs.SupportedMediaTypes(), mediaType)
  437. if !ok {
  438. return []byte{}, fmt.Errorf("unsupported media type %q", mediaType)
  439. }
  440. encoder := scheme.Codecs.EncoderForVersion(info.Serializer, gv)
  441. return runtime.Encode(encoder, obj)
  442. }
  443. // ingressFromManifest reads a .json/yaml file and returns the ingress in it.
  444. func ingressFromManifest(fileName string) (*networkingv1beta1.Ingress, error) {
  445. var ing networkingv1beta1.Ingress
  446. data, err := testfiles.Read(fileName)
  447. if err != nil {
  448. return nil, err
  449. }
  450. json, err := utilyaml.ToJSON(data)
  451. if err != nil {
  452. return nil, err
  453. }
  454. if err := runtime.DecodeInto(scheme.Codecs.UniversalDecoder(), json, &ing); err != nil {
  455. return nil, err
  456. }
  457. return &ing, nil
  458. }
  459. // ingressToManifest generates a yaml file in the given path with the given ingress.
  460. // Assumes that a directory exists at the given path.
  461. func ingressToManifest(ing *networkingv1beta1.Ingress, path string) error {
  462. serialized, err := marshalToYaml(ing, networkingv1beta1.SchemeGroupVersion)
  463. if err != nil {
  464. return fmt.Errorf("failed to marshal ingress %v to YAML: %v", ing, err)
  465. }
  466. if err := ioutil.WriteFile(path, serialized, 0600); err != nil {
  467. return fmt.Errorf("error in writing ingress to file: %s", err)
  468. }
  469. return nil
  470. }
  471. // runCreate runs the required command to create the given ingress.
  472. func (j *TestJig) runCreate(ing *networkingv1beta1.Ingress) (*networkingv1beta1.Ingress, error) {
  473. if j.Class != MulticlusterIngressClassValue {
  474. return j.Client.NetworkingV1beta1().Ingresses(ing.Namespace).Create(context.TODO(), ing, metav1.CreateOptions{})
  475. }
  476. // Use kubemci to create a multicluster ingress.
  477. filePath := framework.TestContext.OutputDir + "/mci.yaml"
  478. if err := ingressToManifest(ing, filePath); err != nil {
  479. return nil, err
  480. }
  481. _, err := framework.RunKubemciWithKubeconfig("create", ing.Name, fmt.Sprintf("--ingress=%s", filePath))
  482. return ing, err
  483. }
  484. // runUpdate runs the required command to update the given ingress.
  485. func (j *TestJig) runUpdate(ing *networkingv1beta1.Ingress) (*networkingv1beta1.Ingress, error) {
  486. if j.Class != MulticlusterIngressClassValue {
  487. return j.Client.NetworkingV1beta1().Ingresses(ing.Namespace).Update(context.TODO(), ing, metav1.UpdateOptions{})
  488. }
  489. // Use kubemci to update a multicluster ingress.
  490. // kubemci does not have an update command. We use "create --force" to update an existing ingress.
  491. filePath := framework.TestContext.OutputDir + "/mci.yaml"
  492. if err := ingressToManifest(ing, filePath); err != nil {
  493. return nil, err
  494. }
  495. _, err := framework.RunKubemciWithKubeconfig("create", ing.Name, fmt.Sprintf("--ingress=%s", filePath), "--force")
  496. return ing, err
  497. }
  498. // Update retrieves the ingress, performs the passed function, and then updates it.
  499. func (j *TestJig) Update(update func(ing *networkingv1beta1.Ingress)) {
  500. var err error
  501. ns, name := j.Ingress.Namespace, j.Ingress.Name
  502. for i := 0; i < 3; i++ {
  503. j.Ingress, err = j.Client.NetworkingV1beta1().Ingresses(ns).Get(context.TODO(), name, metav1.GetOptions{})
  504. if err != nil {
  505. framework.Failf("failed to get ingress %s/%s: %v", ns, name, err)
  506. }
  507. update(j.Ingress)
  508. j.Ingress, err = j.runUpdate(j.Ingress)
  509. if err == nil {
  510. framework.DescribeIng(j.Ingress.Namespace)
  511. return
  512. }
  513. if !apierrors.IsConflict(err) && !apierrors.IsServerTimeout(err) {
  514. framework.Failf("failed to update ingress %s/%s: %v", ns, name, err)
  515. }
  516. }
  517. framework.Failf("too many retries updating ingress %s/%s", ns, name)
  518. }
  519. // AddHTTPS updates the ingress to add this secret for these hosts.
  520. func (j *TestJig) AddHTTPS(secretName string, hosts ...string) {
  521. // TODO: Just create the secret in GetRootCAs once we're watching secrets in
  522. // the ingress controller.
  523. _, cert, _, err := createTLSSecret(j.Client, j.Ingress.Namespace, secretName, hosts...)
  524. framework.ExpectNoError(err)
  525. j.Logger.Infof("Updating ingress %v to also use secret %v for TLS termination", j.Ingress.Name, secretName)
  526. j.Update(func(ing *networkingv1beta1.Ingress) {
  527. ing.Spec.TLS = append(ing.Spec.TLS, networkingv1beta1.IngressTLS{Hosts: hosts, SecretName: secretName})
  528. })
  529. j.RootCAs[secretName] = cert
  530. }
  531. // SetHTTPS updates the ingress to use only this secret for these hosts.
  532. func (j *TestJig) SetHTTPS(secretName string, hosts ...string) {
  533. _, cert, _, err := createTLSSecret(j.Client, j.Ingress.Namespace, secretName, hosts...)
  534. framework.ExpectNoError(err)
  535. j.Logger.Infof("Updating ingress %v to only use secret %v for TLS termination", j.Ingress.Name, secretName)
  536. j.Update(func(ing *networkingv1beta1.Ingress) {
  537. ing.Spec.TLS = []networkingv1beta1.IngressTLS{{Hosts: hosts, SecretName: secretName}}
  538. })
  539. j.RootCAs = map[string][]byte{secretName: cert}
  540. }
  541. // RemoveHTTPS updates the ingress to not use this secret for TLS.
  542. // Note: Does not delete the secret.
  543. func (j *TestJig) RemoveHTTPS(secretName string) {
  544. newTLS := []networkingv1beta1.IngressTLS{}
  545. for _, ingressTLS := range j.Ingress.Spec.TLS {
  546. if secretName != ingressTLS.SecretName {
  547. newTLS = append(newTLS, ingressTLS)
  548. }
  549. }
  550. j.Logger.Infof("Updating ingress %v to not use secret %v for TLS termination", j.Ingress.Name, secretName)
  551. j.Update(func(ing *networkingv1beta1.Ingress) {
  552. ing.Spec.TLS = newTLS
  553. })
  554. delete(j.RootCAs, secretName)
  555. }
  556. // PrepareTLSSecret creates a TLS secret and caches the cert.
  557. func (j *TestJig) PrepareTLSSecret(namespace, secretName string, hosts ...string) error {
  558. _, cert, _, err := createTLSSecret(j.Client, namespace, secretName, hosts...)
  559. if err != nil {
  560. return err
  561. }
  562. j.RootCAs[secretName] = cert
  563. return nil
  564. }
  565. // GetRootCA returns a rootCA from the ingress test jig.
  566. func (j *TestJig) GetRootCA(secretName string) (rootCA []byte) {
  567. var ok bool
  568. rootCA, ok = j.RootCAs[secretName]
  569. if !ok {
  570. framework.Failf("Failed to retrieve rootCAs, no recorded secret by name %v", secretName)
  571. }
  572. return
  573. }
  574. // TryDeleteIngress attempts to delete the ingress resource and logs errors if they occur.
  575. func (j *TestJig) TryDeleteIngress() {
  576. j.tryDeleteGivenIngress(j.Ingress)
  577. }
  578. func (j *TestJig) tryDeleteGivenIngress(ing *networkingv1beta1.Ingress) {
  579. if err := j.runDelete(ing); err != nil {
  580. j.Logger.Infof("Error while deleting the ingress %v/%v with class %s: %v", ing.Namespace, ing.Name, j.Class, err)
  581. }
  582. }
  583. // runDelete runs the required command to delete the given ingress.
  584. func (j *TestJig) runDelete(ing *networkingv1beta1.Ingress) error {
  585. if j.Class != MulticlusterIngressClassValue {
  586. return j.Client.NetworkingV1beta1().Ingresses(ing.Namespace).Delete(context.TODO(), ing.Name, nil)
  587. }
  588. // Use kubemci to delete a multicluster ingress.
  589. filePath := framework.TestContext.OutputDir + "/mci.yaml"
  590. if err := ingressToManifest(ing, filePath); err != nil {
  591. return err
  592. }
  593. _, err := framework.RunKubemciWithKubeconfig("delete", ing.Name, fmt.Sprintf("--ingress=%s", filePath))
  594. return err
  595. }
  596. // getIngressAddressFromKubemci returns the IP address of the given multicluster ingress using kubemci.
  597. // TODO(nikhiljindal): Update this to be able to return hostname as well.
  598. func getIngressAddressFromKubemci(name string) ([]string, error) {
  599. var addresses []string
  600. out, err := framework.RunKubemciCmd("get-status", name)
  601. if err != nil {
  602. return addresses, err
  603. }
  604. ip := findIPv4(out)
  605. if ip != "" {
  606. addresses = append(addresses, ip)
  607. }
  608. return addresses, err
  609. }
  610. // findIPv4 returns the first IPv4 address found in the given string.
  611. func findIPv4(input string) string {
  612. numBlock := "(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])"
  613. regexPattern := numBlock + "\\." + numBlock + "\\." + numBlock + "\\." + numBlock
  614. regEx := regexp.MustCompile(regexPattern)
  615. return regEx.FindString(input)
  616. }
  617. // getIngressAddress returns the ips/hostnames associated with the Ingress.
  618. func getIngressAddress(client clientset.Interface, ns, name, class string) ([]string, error) {
  619. if class == MulticlusterIngressClassValue {
  620. return getIngressAddressFromKubemci(name)
  621. }
  622. ing, err := client.NetworkingV1beta1().Ingresses(ns).Get(context.TODO(), name, metav1.GetOptions{})
  623. if err != nil {
  624. return nil, err
  625. }
  626. var addresses []string
  627. for _, a := range ing.Status.LoadBalancer.Ingress {
  628. if a.IP != "" {
  629. addresses = append(addresses, a.IP)
  630. }
  631. if a.Hostname != "" {
  632. addresses = append(addresses, a.Hostname)
  633. }
  634. }
  635. return addresses, nil
  636. }
  637. // WaitForIngressAddress waits for the Ingress to acquire an address.
  638. func (j *TestJig) WaitForIngressAddress(c clientset.Interface, ns, ingName string, timeout time.Duration) (string, error) {
  639. var address string
  640. err := wait.PollImmediate(10*time.Second, timeout, func() (bool, error) {
  641. ipOrNameList, err := getIngressAddress(c, ns, ingName, j.Class)
  642. if err != nil || len(ipOrNameList) == 0 {
  643. j.Logger.Errorf("Waiting for Ingress %s/%s to acquire IP, error: %v, ipOrNameList: %v", ns, ingName, err, ipOrNameList)
  644. if testutils.IsRetryableAPIError(err) {
  645. return false, nil
  646. }
  647. return false, err
  648. }
  649. address = ipOrNameList[0]
  650. j.Logger.Infof("Found address %s for ingress %s/%s", address, ns, ingName)
  651. return true, nil
  652. })
  653. return address, err
  654. }
  655. func (j *TestJig) pollIngressWithCert(ing *networkingv1beta1.Ingress, address string, knownHosts []string, cert []byte, waitForNodePort bool, timeout time.Duration) error {
  656. // Check that all rules respond to a simple GET.
  657. knownHostsSet := sets.NewString(knownHosts...)
  658. for _, rules := range ing.Spec.Rules {
  659. timeoutClient := &http.Client{Timeout: IngressReqTimeout}
  660. proto := "http"
  661. if knownHostsSet.Has(rules.Host) {
  662. var err error
  663. // Create transport with cert to verify if the server uses the correct one.
  664. timeoutClient.Transport, err = buildTransportWithCA(rules.Host, cert)
  665. if err != nil {
  666. return err
  667. }
  668. proto = "https"
  669. }
  670. for _, p := range rules.IngressRuleValue.HTTP.Paths {
  671. if waitForNodePort {
  672. nodePort := int(p.Backend.ServicePort.IntVal)
  673. if err := j.pollServiceNodePort(ing.Namespace, p.Backend.ServiceName, nodePort); err != nil {
  674. j.Logger.Infof("Error in waiting for nodeport %d on service %v/%v: %s", nodePort, ing.Namespace, p.Backend.ServiceName, err)
  675. return err
  676. }
  677. }
  678. route := fmt.Sprintf("%v://%v%v", proto, address, p.Path)
  679. j.Logger.Infof("Testing route %v host %v with simple GET", route, rules.Host)
  680. if err := PollURL(route, rules.Host, timeout, j.PollInterval, timeoutClient, false); err != nil {
  681. return err
  682. }
  683. }
  684. }
  685. j.Logger.Infof("Finished polling on all rules on ingress %q", ing.Name)
  686. return nil
  687. }
  688. // WaitForIngress waits for the Ingress to get an address.
  689. // WaitForIngress returns when it gets the first 200 response
  690. func (j *TestJig) WaitForIngress(waitForNodePort bool) {
  691. if err := j.WaitForGivenIngressWithTimeout(j.Ingress, waitForNodePort, e2eservice.LoadBalancerPollTimeout); err != nil {
  692. framework.Failf("error in waiting for ingress to get an address: %s", err)
  693. }
  694. }
  695. // WaitForIngressToStable waits for the LB return 100 consecutive 200 responses.
  696. func (j *TestJig) WaitForIngressToStable() {
  697. if err := wait.Poll(10*time.Second, e2eservice.LoadBalancerPropagationTimeoutDefault, func() (bool, error) {
  698. _, err := j.GetDistinctResponseFromIngress()
  699. if err != nil {
  700. return false, nil
  701. }
  702. return true, nil
  703. }); err != nil {
  704. framework.Failf("error in waiting for ingress to stablize: %v", err)
  705. }
  706. }
  707. // WaitForGivenIngressWithTimeout waits till the ingress acquires an IP,
  708. // then waits for its hosts/urls to respond to a protocol check (either
  709. // http or https). If waitForNodePort is true, the NodePort of the Service
  710. // is verified before verifying the Ingress. NodePort is currently a
  711. // requirement for cloudprovider Ingress.
  712. func (j *TestJig) WaitForGivenIngressWithTimeout(ing *networkingv1beta1.Ingress, waitForNodePort bool, timeout time.Duration) error {
  713. // Wait for the loadbalancer IP.
  714. address, err := j.WaitForIngressAddress(j.Client, ing.Namespace, ing.Name, timeout)
  715. if err != nil {
  716. return fmt.Errorf("Ingress failed to acquire an IP address within %v", timeout)
  717. }
  718. var knownHosts []string
  719. var cert []byte
  720. if len(ing.Spec.TLS) > 0 {
  721. knownHosts = ing.Spec.TLS[0].Hosts
  722. cert = j.GetRootCA(ing.Spec.TLS[0].SecretName)
  723. }
  724. return j.pollIngressWithCert(ing, address, knownHosts, cert, waitForNodePort, timeout)
  725. }
  726. // WaitForIngressWithCert waits till the ingress acquires an IP, then waits for its
  727. // hosts/urls to respond to a protocol check (either http or https). If
  728. // waitForNodePort is true, the NodePort of the Service is verified before
  729. // verifying the Ingress. NodePort is currently a requirement for cloudprovider
  730. // Ingress. Hostnames and certificate need to be explicitly passed in.
  731. func (j *TestJig) WaitForIngressWithCert(waitForNodePort bool, knownHosts []string, cert []byte) error {
  732. // Wait for the loadbalancer IP.
  733. address, err := j.WaitForIngressAddress(j.Client, j.Ingress.Namespace, j.Ingress.Name, e2eservice.LoadBalancerPollTimeout)
  734. if err != nil {
  735. return fmt.Errorf("Ingress failed to acquire an IP address within %v", e2eservice.LoadBalancerPollTimeout)
  736. }
  737. return j.pollIngressWithCert(j.Ingress, address, knownHosts, cert, waitForNodePort, e2eservice.LoadBalancerPollTimeout)
  738. }
  739. // VerifyURL polls for the given iterations, in intervals, and fails if the
  740. // given url returns a non-healthy http code even once.
  741. func (j *TestJig) VerifyURL(route, host string, iterations int, interval time.Duration, httpClient *http.Client) error {
  742. for i := 0; i < iterations; i++ {
  743. b, err := SimpleGET(httpClient, route, host)
  744. if err != nil {
  745. framework.Logf(b)
  746. return err
  747. }
  748. j.Logger.Infof("Verified %v with host %v %d times, sleeping for %v", route, host, i, interval)
  749. time.Sleep(interval)
  750. }
  751. return nil
  752. }
  753. func (j *TestJig) pollServiceNodePort(ns, name string, port int) error {
  754. // TODO: Curl all nodes?
  755. u, err := getPortURL(j.Client, ns, name, port)
  756. if err != nil {
  757. return err
  758. }
  759. return PollURL(u, "", 30*time.Second, j.PollInterval, &http.Client{Timeout: IngressReqTimeout}, false)
  760. }
  761. // getSvcNodePort returns the node port for the given service:port.
  762. func getSvcNodePort(client clientset.Interface, ns, name string, svcPort int) (int, error) {
  763. svc, err := client.CoreV1().Services(ns).Get(context.TODO(), name, metav1.GetOptions{})
  764. if err != nil {
  765. return 0, err
  766. }
  767. for _, p := range svc.Spec.Ports {
  768. if p.Port == int32(svcPort) {
  769. if p.NodePort != 0 {
  770. return int(p.NodePort), nil
  771. }
  772. }
  773. }
  774. return 0, fmt.Errorf(
  775. "no node port found for service %v, port %v", name, svcPort)
  776. }
  777. // getPortURL returns the url to a nodeport Service.
  778. func getPortURL(client clientset.Interface, ns, name string, svcPort int) (string, error) {
  779. nodePort, err := getSvcNodePort(client, ns, name, svcPort)
  780. if err != nil {
  781. return "", err
  782. }
  783. // This list of nodes must not include the master, which is marked
  784. // unschedulable, since the master doesn't run kube-proxy. Without
  785. // kube-proxy NodePorts won't work.
  786. var nodes *v1.NodeList
  787. if wait.PollImmediate(poll, framework.SingleCallTimeout, func() (bool, error) {
  788. nodes, err = client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{FieldSelector: fields.Set{
  789. "spec.unschedulable": "false",
  790. }.AsSelector().String()})
  791. if err != nil {
  792. if testutils.IsRetryableAPIError(err) {
  793. return false, nil
  794. }
  795. return false, err
  796. }
  797. return true, nil
  798. }) != nil {
  799. return "", err
  800. }
  801. if len(nodes.Items) == 0 {
  802. return "", fmt.Errorf("Unable to list nodes in cluster")
  803. }
  804. for _, node := range nodes.Items {
  805. for _, address := range node.Status.Addresses {
  806. if address.Type == v1.NodeExternalIP {
  807. if address.Address != "" {
  808. host := net.JoinHostPort(address.Address, fmt.Sprint(nodePort))
  809. return fmt.Sprintf("http://%s", host), nil
  810. }
  811. }
  812. }
  813. }
  814. return "", fmt.Errorf("failed to find external address for service %v", name)
  815. }
  816. // GetIngressNodePorts returns related backend services' nodePorts.
  817. // Current GCE ingress controller allows traffic to the default HTTP backend
  818. // by default, so retrieve its nodePort if includeDefaultBackend is true.
  819. func (j *TestJig) GetIngressNodePorts(includeDefaultBackend bool) []string {
  820. nodePorts := []string{}
  821. svcPorts := j.GetServicePorts(includeDefaultBackend)
  822. for _, svcPort := range svcPorts {
  823. nodePorts = append(nodePorts, strconv.Itoa(int(svcPort.NodePort)))
  824. }
  825. return nodePorts
  826. }
  827. // GetServicePorts returns related backend services' svcPorts.
  828. // Current GCE ingress controller allows traffic to the default HTTP backend
  829. // by default, so retrieve its nodePort if includeDefaultBackend is true.
  830. func (j *TestJig) GetServicePorts(includeDefaultBackend bool) map[string]v1.ServicePort {
  831. svcPorts := make(map[string]v1.ServicePort)
  832. if includeDefaultBackend {
  833. defaultSvc, err := j.Client.CoreV1().Services(metav1.NamespaceSystem).Get(context.TODO(), defaultBackendName, metav1.GetOptions{})
  834. framework.ExpectNoError(err)
  835. svcPorts[defaultBackendName] = defaultSvc.Spec.Ports[0]
  836. }
  837. backendSvcs := []string{}
  838. if j.Ingress.Spec.Backend != nil {
  839. backendSvcs = append(backendSvcs, j.Ingress.Spec.Backend.ServiceName)
  840. }
  841. for _, rule := range j.Ingress.Spec.Rules {
  842. for _, ingPath := range rule.HTTP.Paths {
  843. backendSvcs = append(backendSvcs, ingPath.Backend.ServiceName)
  844. }
  845. }
  846. for _, svcName := range backendSvcs {
  847. svc, err := j.Client.CoreV1().Services(j.Ingress.Namespace).Get(context.TODO(), svcName, metav1.GetOptions{})
  848. framework.ExpectNoError(err)
  849. svcPorts[svcName] = svc.Spec.Ports[0]
  850. }
  851. return svcPorts
  852. }
  853. // ConstructFirewallForIngress returns the expected GCE firewall rule for the ingress resource
  854. func (j *TestJig) ConstructFirewallForIngress(firewallRuleName string, nodeTags []string) *compute.Firewall {
  855. nodePorts := j.GetIngressNodePorts(true)
  856. fw := compute.Firewall{}
  857. fw.Name = firewallRuleName
  858. fw.SourceRanges = framework.TestContext.CloudConfig.Provider.LoadBalancerSrcRanges()
  859. fw.TargetTags = nodeTags
  860. fw.Allowed = []*compute.FirewallAllowed{
  861. {
  862. IPProtocol: "tcp",
  863. Ports: nodePorts,
  864. },
  865. }
  866. return &fw
  867. }
  868. // GetDistinctResponseFromIngress tries GET call to the ingress VIP and return all distinct responses.
  869. func (j *TestJig) GetDistinctResponseFromIngress() (sets.String, error) {
  870. // Wait for the loadbalancer IP.
  871. address, err := j.WaitForIngressAddress(j.Client, j.Ingress.Namespace, j.Ingress.Name, e2eservice.LoadBalancerPollTimeout)
  872. if err != nil {
  873. framework.Failf("Ingress failed to acquire an IP address within %v", e2eservice.LoadBalancerPollTimeout)
  874. }
  875. responses := sets.NewString()
  876. timeoutClient := &http.Client{Timeout: IngressReqTimeout}
  877. for i := 0; i < 100; i++ {
  878. url := fmt.Sprintf("http://%v", address)
  879. res, err := SimpleGET(timeoutClient, url, "")
  880. if err != nil {
  881. j.Logger.Errorf("Failed to GET %q. Got responses: %q: %v", url, res, err)
  882. return responses, err
  883. }
  884. responses.Insert(res)
  885. }
  886. return responses, nil
  887. }
  888. // NginxIngressController manages implementation details of Ingress on Nginx.
  889. type NginxIngressController struct {
  890. Ns string
  891. rc *v1.ReplicationController
  892. pod *v1.Pod
  893. Client clientset.Interface
  894. lbSvc *v1.Service
  895. }
  896. // Init initializes the NginxIngressController
  897. func (cont *NginxIngressController) Init() {
  898. // Set up a LoadBalancer service in front of nginx ingress controller and pass it via
  899. // --publish-service flag (see <IngressManifestPath>/nginx/rc.yaml) to make it work in private
  900. // clusters, i.e. clusters where nodes don't have public IPs.
  901. framework.Logf("Creating load balancer service for nginx ingress controller")
  902. serviceJig := e2eservice.NewTestJig(cont.Client, cont.Ns, "nginx-ingress-lb")
  903. _, err := serviceJig.CreateTCPService(func(svc *v1.Service) {
  904. svc.Spec.Type = v1.ServiceTypeLoadBalancer
  905. svc.Spec.Selector = map[string]string{"k8s-app": "nginx-ingress-lb"}
  906. svc.Spec.Ports = []v1.ServicePort{
  907. {Name: "http", Port: 80},
  908. {Name: "https", Port: 443},
  909. {Name: "stats", Port: 18080}}
  910. })
  911. framework.ExpectNoError(err)
  912. cont.lbSvc, err = serviceJig.WaitForLoadBalancer(e2eservice.GetServiceLoadBalancerCreationTimeout(cont.Client))
  913. framework.ExpectNoError(err)
  914. read := func(file string) string {
  915. return string(testfiles.ReadOrDie(filepath.Join(IngressManifestPath, "nginx", file)))
  916. }
  917. framework.Logf("initializing nginx ingress controller")
  918. framework.RunKubectlOrDieInput(cont.Ns, read("rc.yaml"), "create", "-f", "-", fmt.Sprintf("--namespace=%v", cont.Ns))
  919. rc, err := cont.Client.CoreV1().ReplicationControllers(cont.Ns).Get(context.TODO(), "nginx-ingress-controller", metav1.GetOptions{})
  920. framework.ExpectNoError(err)
  921. cont.rc = rc
  922. framework.Logf("waiting for pods with label %v", rc.Spec.Selector)
  923. sel := labels.SelectorFromSet(labels.Set(rc.Spec.Selector))
  924. framework.ExpectNoError(testutils.WaitForPodsWithLabelRunning(cont.Client, cont.Ns, sel))
  925. pods, err := cont.Client.CoreV1().Pods(cont.Ns).List(context.TODO(), metav1.ListOptions{LabelSelector: sel.String()})
  926. framework.ExpectNoError(err)
  927. if len(pods.Items) == 0 {
  928. framework.Failf("Failed to find nginx ingress controller pods with selector %v", sel)
  929. }
  930. cont.pod = &pods.Items[0]
  931. framework.Logf("ingress controller running in pod %v", cont.pod.Name)
  932. }
  933. // TearDown cleans up the NginxIngressController.
  934. func (cont *NginxIngressController) TearDown() {
  935. if cont.lbSvc == nil {
  936. framework.Logf("No LoadBalancer service created, no cleanup necessary")
  937. return
  938. }
  939. e2eservice.WaitForServiceDeletedWithFinalizer(cont.Client, cont.Ns, cont.lbSvc.Name)
  940. }
  941. func generateBacksideHTTPSIngressSpec(ns string) *networkingv1beta1.Ingress {
  942. return &networkingv1beta1.Ingress{
  943. ObjectMeta: metav1.ObjectMeta{
  944. Name: "echoheaders-https",
  945. Namespace: ns,
  946. },
  947. Spec: networkingv1beta1.IngressSpec{
  948. // Note kubemci requires a default backend.
  949. Backend: &networkingv1beta1.IngressBackend{
  950. ServiceName: "echoheaders-https",
  951. ServicePort: intstr.IntOrString{
  952. Type: intstr.Int,
  953. IntVal: 443,
  954. },
  955. },
  956. },
  957. }
  958. }
  959. func generateBacksideHTTPSServiceSpec() *v1.Service {
  960. return &v1.Service{
  961. ObjectMeta: metav1.ObjectMeta{
  962. Name: "echoheaders-https",
  963. Annotations: map[string]string{
  964. ServiceApplicationProtocolKey: `{"my-https-port":"HTTPS"}`,
  965. },
  966. },
  967. Spec: v1.ServiceSpec{
  968. Ports: []v1.ServicePort{{
  969. Name: "my-https-port",
  970. Protocol: v1.ProtocolTCP,
  971. Port: 443,
  972. TargetPort: intstr.FromString("echo-443"),
  973. }},
  974. Selector: map[string]string{
  975. "app": "echoheaders-https",
  976. },
  977. Type: v1.ServiceTypeNodePort,
  978. },
  979. }
  980. }
  981. func generateBacksideHTTPSDeploymentSpec() *appsv1.Deployment {
  982. return &appsv1.Deployment{
  983. ObjectMeta: metav1.ObjectMeta{
  984. Name: "echoheaders-https",
  985. },
  986. Spec: appsv1.DeploymentSpec{
  987. Selector: &metav1.LabelSelector{MatchLabels: map[string]string{
  988. "app": "echoheaders-https",
  989. }},
  990. Template: v1.PodTemplateSpec{
  991. ObjectMeta: metav1.ObjectMeta{
  992. Labels: map[string]string{
  993. "app": "echoheaders-https",
  994. },
  995. },
  996. Spec: v1.PodSpec{
  997. Containers: []v1.Container{
  998. {
  999. Name: "echoheaders-https",
  1000. Image: imageutils.GetE2EImage(imageutils.EchoServer),
  1001. Ports: []v1.ContainerPort{{
  1002. ContainerPort: 8443,
  1003. Name: "echo-443",
  1004. }},
  1005. },
  1006. },
  1007. },
  1008. },
  1009. },
  1010. }
  1011. }
  1012. // SetUpBacksideHTTPSIngress sets up deployment, service and ingress with backside HTTPS configured.
  1013. func (j *TestJig) SetUpBacksideHTTPSIngress(cs clientset.Interface, namespace string, staticIPName string) (*appsv1.Deployment, *v1.Service, *networkingv1beta1.Ingress, error) {
  1014. deployCreated, err := cs.AppsV1().Deployments(namespace).Create(context.TODO(), generateBacksideHTTPSDeploymentSpec(), metav1.CreateOptions{})
  1015. if err != nil {
  1016. return nil, nil, nil, err
  1017. }
  1018. svcCreated, err := cs.CoreV1().Services(namespace).Create(context.TODO(), generateBacksideHTTPSServiceSpec(), metav1.CreateOptions{})
  1019. if err != nil {
  1020. return nil, nil, nil, err
  1021. }
  1022. ingToCreate := generateBacksideHTTPSIngressSpec(namespace)
  1023. if staticIPName != "" {
  1024. if ingToCreate.Annotations == nil {
  1025. ingToCreate.Annotations = map[string]string{}
  1026. }
  1027. ingToCreate.Annotations[IngressStaticIPKey] = staticIPName
  1028. }
  1029. ingCreated, err := j.runCreate(ingToCreate)
  1030. if err != nil {
  1031. return nil, nil, nil, err
  1032. }
  1033. return deployCreated, svcCreated, ingCreated, nil
  1034. }
  1035. // DeleteTestResource deletes given deployment, service and ingress.
  1036. func (j *TestJig) DeleteTestResource(cs clientset.Interface, deploy *appsv1.Deployment, svc *v1.Service, ing *networkingv1beta1.Ingress) []error {
  1037. var errs []error
  1038. if ing != nil {
  1039. if err := j.runDelete(ing); err != nil {
  1040. errs = append(errs, fmt.Errorf("error while deleting ingress %s/%s: %v", ing.Namespace, ing.Name, err))
  1041. }
  1042. }
  1043. if svc != nil {
  1044. if err := cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, nil); err != nil {
  1045. errs = append(errs, fmt.Errorf("error while deleting service %s/%s: %v", svc.Namespace, svc.Name, err))
  1046. }
  1047. }
  1048. if deploy != nil {
  1049. if err := cs.AppsV1().Deployments(deploy.Namespace).Delete(context.TODO(), deploy.Name, nil); err != nil {
  1050. errs = append(errs, fmt.Errorf("error while deleting deployment %s/%s: %v", deploy.Namespace, deploy.Name, err))
  1051. }
  1052. }
  1053. return errs
  1054. }