gke_environment_test.go 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432
  1. /*
  2. Copyright 2017 The Kubernetes Authors.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. */
  13. package e2e_node
  14. import (
  15. "bytes"
  16. "fmt"
  17. "io/ioutil"
  18. "os"
  19. "os/exec"
  20. "regexp"
  21. "strconv"
  22. "strings"
  23. "k8s.io/kubernetes/test/e2e/framework"
  24. imageutils "k8s.io/kubernetes/test/utils/image"
  25. "github.com/blang/semver"
  26. . "github.com/onsi/ginkgo"
  27. )
  28. // checkProcess checks whether there's a process whose command line contains
  29. // the specified pattern and whose parent process id is ppid using the
  30. // pre-built information in cmdToProcessMap.
  31. func checkProcess(pattern string, ppid int, cmdToProcessMap map[string][]process) error {
  32. for cmd, processes := range cmdToProcessMap {
  33. if !strings.Contains(cmd, pattern) {
  34. continue
  35. }
  36. for _, p := range processes {
  37. if p.ppid == ppid {
  38. return nil
  39. }
  40. }
  41. }
  42. return fmt.Errorf("failed to find the process whose cmdline contains %q with ppid = %d", pattern, ppid)
  43. }
  44. // checkIPTables checks whether the functionality required by kube-proxy works
  45. // in iptables.
  46. func checkIPTables() (err error) {
  47. cmds := [][]string{
  48. {"iptables", "-N", "KUBE-PORTALS-HOST", "-t", "nat"},
  49. {"iptables", "-I", "OUTPUT", "-t", "nat", "-m", "comment", "--comment", "ClusterIPs", "-j", "KUBE-PORTALS-HOST"},
  50. {"iptables", "-A", "KUBE-PORTALS-HOST", "-t", "nat", "-m", "comment", "--comment", "test-1:", "-p", "tcp", "-m", "tcp", "--dport", "443", "-d", "10.0.0.1/32", "-j", "DNAT", "--to-destination", "10.240.0.1:11111"},
  51. {"iptables", "-C", "KUBE-PORTALS-HOST", "-t", "nat", "-m", "comment", "--comment", "test-1:", "-p", "tcp", "-m", "tcp", "--dport", "443", "-d", "10.0.0.1/32", "-j", "DNAT", "--to-destination", "10.240.0.1:11111"},
  52. {"iptables", "-A", "KUBE-PORTALS-HOST", "-t", "nat", "-m", "comment", "--comment", "test-2:", "-p", "tcp", "-m", "tcp", "--dport", "80", "-d", "10.0.0.1/32", "-j", "REDIRECT", "--to-ports", "22222"},
  53. {"iptables", "-C", "KUBE-PORTALS-HOST", "-t", "nat", "-m", "comment", "--comment", "test-2:", "-p", "tcp", "-m", "tcp", "--dport", "80", "-d", "10.0.0.1/32", "-j", "REDIRECT", "--to-ports", "22222"},
  54. }
  55. cleanupCmds := [][]string{
  56. {"iptables", "-F", "KUBE-PORTALS-HOST", "-t", "nat"},
  57. {"iptables", "-D", "OUTPUT", "-t", "nat", "-m", "comment", "--comment", "ClusterIPs", "-j", "KUBE-PORTALS-HOST"},
  58. {"iptables", "-X", "KUBE-PORTALS-HOST", "-t", "nat"},
  59. }
  60. defer func() {
  61. for _, cmd := range cleanupCmds {
  62. if _, cleanupErr := runCommand(cmd...); cleanupErr != nil && err == nil {
  63. err = cleanupErr
  64. return
  65. }
  66. }
  67. }()
  68. for _, cmd := range cmds {
  69. if _, err := runCommand(cmd...); err != nil {
  70. return err
  71. }
  72. }
  73. return
  74. }
  75. // checkPublicGCR checks the access to the public Google Container Registry by
  76. // pulling the busybox image.
  77. func checkPublicGCR() error {
  78. const image = "k8s.gcr.io/busybox"
  79. output, err := runCommand("docker", "images", "-q", image)
  80. if len(output) != 0 {
  81. if _, err := runCommand("docker", "rmi", "-f", image); err != nil {
  82. return err
  83. }
  84. }
  85. output, err = runCommand("docker", "pull", image)
  86. if len(output) == 0 {
  87. return fmt.Errorf("failed to pull %s", image)
  88. }
  89. if _, err = runCommand("docker", "rmi", "-f", image); err != nil {
  90. return err
  91. }
  92. return nil
  93. }
  94. // checkDockerConfig runs docker's check-config.sh script and ensures that all
  95. // expected kernel configs are enabled.
  96. func checkDockerConfig() error {
  97. var (
  98. re = regexp.MustCompile("\x1B\\[([0-9]{1,2}(;[0-9]{1,2})?)?[mGK]")
  99. bins = []string{
  100. "/usr/share/docker.io/contrib/check-config.sh",
  101. "/usr/share/docker/contrib/check-config.sh",
  102. }
  103. whitelist = map[string]bool{
  104. "CONFIG_MEMCG_SWAP_ENABLED": true,
  105. "CONFIG_RT_GROUP_SCHED": true,
  106. "CONFIG_EXT3_FS": true,
  107. "CONFIG_EXT3_FS_XATTR": true,
  108. "CONFIG_EXT3_FS_POSIX_ACL": true,
  109. "CONFIG_EXT3_FS_SECURITY": true,
  110. "/dev/zfs": true,
  111. "zfs command": true,
  112. "zpool command": true,
  113. }
  114. missing = map[string]bool{}
  115. )
  116. // Whitelists CONFIG_DEVPTS_MULTIPLE_INSTANCES (meaning allowing it to be
  117. // absent) if the kernel version is >= 4.8, because this option has been
  118. // removed from the 4.8 kernel.
  119. kernelVersion, err := getKernelVersion()
  120. if err != nil {
  121. return err
  122. }
  123. if kernelVersion.GTE(semver.MustParse("4.8.0")) {
  124. whitelist["CONFIG_DEVPTS_MULTIPLE_INSTANCES"] = true
  125. }
  126. for _, bin := range bins {
  127. if _, err := os.Stat(bin); os.IsNotExist(err) {
  128. continue
  129. }
  130. // We don't check the return code because it's OK if the script returns
  131. // a non-zero exit code just because the configs in the whitelist are
  132. // missing.
  133. output, _ := runCommand(bin)
  134. for _, line := range strings.Split(output, "\n") {
  135. if !strings.Contains(line, "missing") {
  136. continue
  137. }
  138. line = re.ReplaceAllString(line, "")
  139. fields := strings.Split(line, ":")
  140. if len(fields) != 2 {
  141. continue
  142. }
  143. key := strings.TrimFunc(fields[0], func(c rune) bool {
  144. return c == ' ' || c == '-'
  145. })
  146. if _, found := whitelist[key]; !found {
  147. missing[key] = true
  148. }
  149. }
  150. if len(missing) != 0 {
  151. return fmt.Errorf("missing docker config: %v", missing)
  152. }
  153. break
  154. }
  155. return nil
  156. }
  157. // checkDockerNetworkClient checks client networking by pinging an external IP
  158. // address from a container.
  159. func checkDockerNetworkClient() error {
  160. imageName := imageutils.GetE2EImage(imageutils.BusyBox)
  161. output, err := runCommand("docker", "run", "--rm", imageName, "sh", "-c", "ping -w 5 -q google.com")
  162. if err != nil {
  163. return err
  164. }
  165. if !strings.Contains(output, `0% packet loss`) {
  166. return fmt.Errorf("failed to ping from container: %s", output)
  167. }
  168. return nil
  169. }
  170. // checkDockerNetworkServer checks server networking by running an echo server
  171. // within a container and accessing it from outside.
  172. func checkDockerNetworkServer() error {
  173. const (
  174. imageName = "k8s.gcr.io/nginx:1.7.9"
  175. hostAddr = "127.0.0.1"
  176. hostPort = "8088"
  177. containerPort = "80"
  178. containerID = "nginx"
  179. message = "Welcome to nginx!"
  180. )
  181. var (
  182. portMapping = fmt.Sprintf("%s:%s", hostPort, containerPort)
  183. host = fmt.Sprintf("http://%s:%s", hostAddr, hostPort)
  184. )
  185. runCommand("docker", "rm", "-f", containerID)
  186. if _, err := runCommand("docker", "run", "-d", "--name", containerID, "-p", portMapping, imageName); err != nil {
  187. return err
  188. }
  189. output, err := runCommand("curl", host)
  190. if err != nil {
  191. return err
  192. }
  193. if !strings.Contains(output, message) {
  194. return fmt.Errorf("failed to connect to container")
  195. }
  196. // Clean up
  197. if _, err = runCommand("docker", "rm", "-f", containerID); err != nil {
  198. return err
  199. }
  200. if _, err = runCommand("docker", "rmi", imageName); err != nil {
  201. return err
  202. }
  203. return nil
  204. }
  205. // checkDockerAppArmor checks whether AppArmor is enabled and has the
  206. // "docker-default" profile.
  207. func checkDockerAppArmor() error {
  208. buf, err := ioutil.ReadFile("/sys/module/apparmor/parameters/enabled")
  209. if err != nil {
  210. return err
  211. }
  212. if string(buf) != "Y\n" {
  213. return fmt.Errorf("apparmor module is not loaded")
  214. }
  215. // Checks that the "docker-default" profile is loaded and enforced.
  216. buf, err = ioutil.ReadFile("/sys/kernel/security/apparmor/profiles")
  217. if err != nil {
  218. return err
  219. }
  220. if !strings.Contains(string(buf), "docker-default (enforce)") {
  221. return fmt.Errorf("'docker-default' profile is not loaded and enforced")
  222. }
  223. // Checks that the `apparmor_parser` binary is present.
  224. _, err = exec.LookPath("apparmor_parser")
  225. if err != nil {
  226. return fmt.Errorf("'apparmor_parser' is not in directories named by the PATH env")
  227. }
  228. return nil
  229. }
  230. // checkDockerSeccomp checks whether the Docker supports seccomp.
  231. func checkDockerSeccomp() error {
  232. const (
  233. seccompProfileFileName = "/tmp/no_mkdir.json"
  234. seccompProfile = `{
  235. "defaultAction": "SCMP_ACT_ALLOW",
  236. "syscalls": [
  237. {
  238. "name": "mkdir",
  239. "action": "SCMP_ACT_ERRNO"
  240. }
  241. ]}`
  242. image = "gcr.io/google-appengine/debian8:2017-06-07-171918"
  243. )
  244. if err := ioutil.WriteFile(seccompProfileFileName, []byte(seccompProfile), 0644); err != nil {
  245. return err
  246. }
  247. // Starts a container with no seccomp profile and ensures that unshare
  248. // succeeds.
  249. _, err := runCommand("docker", "run", "--rm", "-i", "--security-opt", "seccomp=unconfined", image, "unshare", "-r", "whoami")
  250. if err != nil {
  251. return err
  252. }
  253. // Starts a container with the default seccomp profile and ensures that
  254. // unshare (a blacklisted system call in the default profile) fails.
  255. cmd := []string{"docker", "run", "--rm", "-i", image, "unshare", "-r", "whoami"}
  256. _, err = runCommand(cmd...)
  257. if err == nil {
  258. return fmt.Errorf("%q did not fail as expected", strings.Join(cmd, " "))
  259. }
  260. // Starts a container with a custom seccomp profile that blacklists mkdir
  261. // and ensures that unshare succeeds.
  262. _, err = runCommand("docker", "run", "--rm", "-i", "--security-opt", fmt.Sprintf("seccomp=%s", seccompProfileFileName), image, "unshare", "-r", "whoami")
  263. if err != nil {
  264. return err
  265. }
  266. // Starts a container with a custom seccomp profile that blacklists mkdir
  267. // and ensures that mkdir fails.
  268. cmd = []string{"docker", "run", "--rm", "-i", "--security-opt", fmt.Sprintf("seccomp=%s", seccompProfileFileName), image, "mkdir", "-p", "/tmp/foo"}
  269. _, err = runCommand(cmd...)
  270. if err == nil {
  271. return fmt.Errorf("%q did not fail as expected", strings.Join(cmd, " "))
  272. }
  273. return nil
  274. }
  275. // checkDockerStorageDriver checks whether the current storage driver used by
  276. // Docker is overlay.
  277. func checkDockerStorageDriver() error {
  278. output, err := runCommand("docker", "info")
  279. if err != nil {
  280. return err
  281. }
  282. for _, line := range strings.Split(string(output), "\n") {
  283. if !strings.Contains(line, "Storage Driver:") {
  284. continue
  285. }
  286. if !strings.Contains(line, "overlay") {
  287. return fmt.Errorf("storage driver is not 'overlay': %s", line)
  288. }
  289. return nil
  290. }
  291. return fmt.Errorf("failed to find storage driver")
  292. }
  293. var _ = framework.KubeDescribe("GKE system requirements [NodeConformance][Feature:GKEEnv][NodeFeature:GKEEnv]", func() {
  294. BeforeEach(func() {
  295. framework.RunIfSystemSpecNameIs("gke")
  296. })
  297. It("The required processes should be running", func() {
  298. cmdToProcessMap, err := getCmdToProcessMap()
  299. framework.ExpectNoError(err)
  300. for _, p := range []struct {
  301. cmd string
  302. ppid int
  303. }{
  304. {"google_accounts_daemon", 1},
  305. {"google_clock_skew_daemon", 1},
  306. {"google_ip_forwarding_daemon", 1},
  307. } {
  308. framework.ExpectNoError(checkProcess(p.cmd, p.ppid, cmdToProcessMap))
  309. }
  310. })
  311. It("The iptable rules should work (required by kube-proxy)", func() {
  312. framework.ExpectNoError(checkIPTables())
  313. })
  314. It("The GCR is accessible", func() {
  315. framework.ExpectNoError(checkPublicGCR())
  316. })
  317. It("The docker configuration validation should pass", func() {
  318. framework.RunIfContainerRuntimeIs("docker")
  319. framework.ExpectNoError(checkDockerConfig())
  320. })
  321. It("The docker container network should work", func() {
  322. framework.RunIfContainerRuntimeIs("docker")
  323. framework.ExpectNoError(checkDockerNetworkServer())
  324. framework.ExpectNoError(checkDockerNetworkClient())
  325. })
  326. It("The docker daemon should support AppArmor and seccomp", func() {
  327. framework.RunIfContainerRuntimeIs("docker")
  328. framework.ExpectNoError(checkDockerAppArmor())
  329. framework.ExpectNoError(checkDockerSeccomp())
  330. })
  331. It("The docker storage driver should work", func() {
  332. framework.Skipf("GKE does not currently require overlay")
  333. framework.ExpectNoError(checkDockerStorageDriver())
  334. })
  335. })
  336. // getPPID returns the PPID for the pid.
  337. func getPPID(pid int) (int, error) {
  338. statusFile := "/proc/" + strconv.Itoa(pid) + "/status"
  339. content, err := ioutil.ReadFile(statusFile)
  340. if err != nil {
  341. return 0, err
  342. }
  343. for _, line := range strings.Split(string(content), "\n") {
  344. if !strings.HasPrefix(line, "PPid:") {
  345. continue
  346. }
  347. s := strings.TrimSpace(strings.TrimPrefix(line, "PPid:"))
  348. ppid, err := strconv.Atoi(s)
  349. if err != nil {
  350. return 0, err
  351. }
  352. return ppid, nil
  353. }
  354. return 0, fmt.Errorf("no PPid in %s", statusFile)
  355. }
  356. // process contains a process ID and its parent's process ID.
  357. type process struct {
  358. pid int
  359. ppid int
  360. }
  361. // getCmdToProcessMap returns a mapping from the process command line to its
  362. // process ids.
  363. func getCmdToProcessMap() (map[string][]process, error) {
  364. root, err := os.Open("/proc")
  365. if err != nil {
  366. return nil, err
  367. }
  368. defer root.Close()
  369. dirs, err := root.Readdirnames(0)
  370. if err != nil {
  371. return nil, err
  372. }
  373. result := make(map[string][]process)
  374. for _, dir := range dirs {
  375. pid, err := strconv.Atoi(dir)
  376. if err != nil {
  377. continue
  378. }
  379. ppid, err := getPPID(pid)
  380. if err != nil {
  381. continue
  382. }
  383. content, err := ioutil.ReadFile("/proc/" + dir + "/cmdline")
  384. if err != nil || len(content) == 0 {
  385. continue
  386. }
  387. cmd := string(bytes.Replace(content, []byte("\x00"), []byte(" "), -1))
  388. result[cmd] = append(result[cmd], process{pid, ppid})
  389. }
  390. return result, nil
  391. }
  392. // getKernelVersion returns the kernel version in the semantic version format.
  393. func getKernelVersion() (*semver.Version, error) {
  394. output, err := runCommand("uname", "-r")
  395. if err != nil {
  396. return nil, err
  397. }
  398. // An example 'output' could be "4.13.0-1001-gke".
  399. v := strings.TrimSpace(strings.Split(output, "-")[0])
  400. kernelVersion, err := semver.Make(v)
  401. if err != nil {
  402. return nil, fmt.Errorf("failed to convert %q to semantic version: %s", v, err)
  403. }
  404. return &kernelVersion, nil
  405. }