diff --git a/.circleci/config.yml b/.circleci/config.yml index c71dcb6..209ff36 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,7 +4,7 @@ linuxkit_pkg_build: &linuxkit_pkg_build steps: - run: name: Configure $PATH - command: echo 'export PATH=/workspace/bin:$PATH' >> $BASH_ENV + command: echo 'export PATH=/workspace/bin-Linux:$PATH' >> $BASH_ENV - run: name: Install packages # ca-certificates are needed for attach_workspace (and git over https) @@ -38,7 +38,7 @@ image_build: &image_build steps: - run: name: Configure $PATH - command: echo 'export PATH=/workspace/bin:$PATH' >> $BASH_ENV + command: echo 'export PATH=/workspace/bin-Linux:$PATH' >> $BASH_ENV - run: name: Install packages # ca-certificates are needed for attach_workspace (and git over https) @@ -83,9 +83,73 @@ image_build: &image_build name: Build images command: | mkdir -p /workspace/images/kube-$KUBE_RUNTIME-$KUBE_NETWORK - # KUBE_FORMATS="iso-efi iso-bios" are much slower (especially for RUNTIME=docker) to build than tar. - # So for now just build tar files. - make KUBE_FORMATS="tar" master node + make KUBE_FORMATS="iso-bios" master node + mv kube-master*.iso kube-node*.iso /workspace/images/kube-$KUBE_RUNTIME-$KUBE_NETWORK + - persist_to_workspace: + root: /workspace + paths: images + +image_boot: &image_boot + #machine: + # enabled: true + # image: circleci/classic:latest # default Ubuntu 14.04 image + macos: + xcode: "9.0" + steps: + - run: + name: Configure $PATH + #command: echo 'export PATH=~/workspace/bin-Linux:$PATH' >> $BASH_ENV + command: echo 'export PATH=~/workspace/bin-Darwin:$PATH' >> $BASH_ENV + - run: + name: Install packages + # ca-certificates are needed for attach_workspace (and git over https) + #command: sudo apt-get update && sudo apt-get install -y ca-certificates curl expect git make openssh-client procps qemu xorriso + command: brew install xorriso + - attach_workspace: + at: ~/workspace + - checkout + - run: + name: Versions + command: | + set -x + ~/workspace/bin-Darwin/linuxkit version + ~/workspace/bin-Darwin/vpnkit version + - run: + name: Test boot + command: | + set -x + # Add ssh keys to the image. + # TODO could this be integrated into linuxkit/mkimage-iso-* + # such that all those options do not need to be repeated + # here? e.g. perhaps this could be made to work: + # - Create tar with /root/.ssh/authorized_keys=~/workspace/ssh/id_rsa.pub in it + # - cat $tar | docker run -v ~/workspace/images/kube-$KUBE_RUNTIME-$KUBE_NETWORK/kube-master.iso:/input.iso linuxkit/mkimage-iso-bios -add-to /input.iso + # + # Compared with the options used i nmkimage-iso-bios to + # build the original dropped -J -joliet-long since xorriso + # is more particular about it than genisoimage producing + # lots of: + # + # Cannot add /bin/ls to Joliet tree. Symlinks can only be added to a Rock Ridge tree. + + id_rsa=~/workspace/ssh/id_rsa.pub + xorriso -indev ~/workspace/images/kube-$KUBE_RUNTIME-$KUBE_NETWORK/kube-master.iso \ + -outdev kube-master.iso \ + -as genisoimage \ + -l -R \ + -c isolinux/boot.cat \ + -b isolinux/isolinux.bin \ + -no-emul-boot \ + -boot-load-size 4 \ + -boot-info-table \ + -input-charset utf8 \ + -- \ + -pathspecs on \ + -add /root/.ssh/authorized_keys=$id_rsa + + #cp ~/workspace/images/kube-$KUBE_RUNTIME-$KUBE_NETWORK/kube-{master,node}.iso . + #cp ~/workspace/images/kube-$KUBE_RUNTIME-$KUBE_NETWORK/kube-master.iso . + SSHOPTS="-i ~/workspace/ssh/id_rsa" ./test.exp version: 2 jobs: @@ -95,40 +159,59 @@ jobs: steps: - run: name: Create workspace - command: mkdir -p /workspace/bin + command: mkdir -p /workspace/bin-Linux /workspace/bin-Darwin - run: name: Install packages - command: apt-get update && apt-get install -y ca-certificates curl + command: apt-get update && apt-get install -y ca-certificates curl openssh-client - run: name: Fetch binaries command: | + set -x curl -fsSL -o /tmp/docker.tgz https://download.docker.com/linux/static/stable/x86_64/docker-17.06.2-ce.tgz - tar xfO /tmp/docker.tgz docker/docker > /workspace/bin/docker - curl -fsSL -o /workspace/bin/linuxkit https://362-46932243-gh.circle-artifacts.com/0/linuxkit-linux-amd64 - curl -fsSL -o /workspace/bin/manifest-tool https://github.com/estesp/manifest-tool/releases/download/v0.7.0/manifest-tool-linux-amd64 - curl -fsSL -o /workspace/bin/notary https://github.com/theupdateframework/notary/releases/download/v0.4.3/notary-Linux-amd64 + tar xfO /tmp/docker.tgz docker/docker > /workspace/bin-Linux/docker + curl -fsSL -o /workspace/bin-Linux/linuxkit https://362-46932243-gh.circle-artifacts.com/0/linuxkit-linux-amd64 + curl -fsSL -o /workspace/bin-Darwin/linuxkit https://362-46932243-gh.circle-artifacts.com/0/linuxkit-darwin-amd64 + curl -fsSL -o /workspace/bin-Linux/manifest-tool https://github.com/estesp/manifest-tool/releases/download/v0.7.0/manifest-tool-linux-amd64 + curl -fsSL -o /workspace/bin-Linux/notary https://github.com/theupdateframework/notary/releases/download/v0.4.3/notary-Linux-amd64 + + curl -fsSL -o /workspace/bin-Darwin/hyperkit https://452-55985023-gh.circle-artifacts.com/0/Users/distiller/hyperkit/build/hyperkit + curl -fsSL -o /tmp/vpnkit.tgz https://1066-58395340-gh.circle-artifacts.com/0/Users/distiller/vpnkit/vpnkit.tgz + tar xfO /tmp/vpnkit.tgz Contents/Resources/bin/vpnkit > /workspace/bin-Darwin/vpnkit echo "Downloaded:" - sha256sum /workspace/bin/* + sha256sum /workspace/bin-Linux/* /workspace/bin-Darwin/* echo echo "Checking checksums" sha256sum -c <> $BASH_ENV + command: echo 'export PATH=/workspace/bin-Linux:$PATH' >> $BASH_ENV - run: name: Install packages # ca-certificates are needed for attach_workspace (and git over https) @@ -196,36 +279,60 @@ jobs: <<: *linuxkit_pkg_build pkg-cri-containerd: <<: *linuxkit_pkg_build - pkg-kube-e2e-test: - <<: *linuxkit_pkg_build + #pkg-kube-e2e-test: + # <<: *linuxkit_pkg_build - pkg-kubernetes-docker-image-cache-common: - <<: *linuxkit_pkg_build - pkg-kubernetes-docker-image-cache-control-plane: - <<: *linuxkit_pkg_build + #pkg-kubernetes-docker-image-cache-common: + # <<: *linuxkit_pkg_build + #pkg-kubernetes-docker-image-cache-control-plane: + # <<: *linuxkit_pkg_build - image-docker-weave: - <<: *image_build - # Needs to be configured/enabled by CircleCI person - #resource_class: large - environment: - - KUBE_RUNTIME: docker - - KUBE_NETWORK: weave - image-docker-bridge: - <<: *image_build - # Needs to be configured/enabled by CircleCI person - #resource_class: large - environment: - - KUBE_RUNTIME: docker - - KUBE_NETWORK: bridge + #image-docker-weave: + # <<: *image_build + # # Needs to be configured/enabled by CircleCI person + # #resource_class: large + # environment: + # - KUBE_RUNTIME: docker + # - KUBE_NETWORK: weave + #image-docker-bridge: + # <<: *image_build + # # Needs to be configured/enabled by CircleCI person + # #resource_class: large + # environment: + # - KUBE_RUNTIME: docker + # - KUBE_NETWORK: bridge - image-cri-containerd-weave: + #image-cri-containerd-weave: + # <<: *image_build + # environment: + # - KUBE_RUNTIME: cri-containerd + # - KUBE_NETWORK: weave + image-cri-containerd-bridge: <<: *image_build environment: - KUBE_RUNTIME: cri-containerd - - KUBE_NETWORK: weave - image-cri-containerd-bridge: - <<: *image_build + - KUBE_NETWORK: bridge + + #boot-docker-weave: + # <<: *image_boot + # environment: + # - KUBE_RUNTIME: docker + # - KUBE_NETWORK: weave + + #boot-docker-bridge: + # <<: *image_boot + # environment: + # - KUBE_RUNTIME: docker + # - KUBE_NETWORK: bridge + + #boot-cri-containerd-weave: + # <<: *image_boot + # environment: + # - KUBE_RUNTIME: cri-containerd + # - KUBE_NETWORK: weave + + boot-cri-containerd-bridge: + <<: *image_boot environment: - KUBE_RUNTIME: cri-containerd - KUBE_NETWORK: bridge @@ -236,7 +343,7 @@ jobs: steps: - run: name: Configure $PATH - command: echo 'export PATH=/workspace/bin:$PATH' >> $BASH_ENV + command: echo 'export PATH=/workspace/bin-Linux:$PATH' >> $BASH_ENV - run: name: Install packages # ca-certificates are needed for attach_workspace (and git over https) @@ -268,6 +375,8 @@ jobs: exit 0 fi + false # Debug branch, not expected to get this far + docker login -u $DOCKER_USER -p $DOCKER_PASS mkdir -p ~/.docker/trust/private cp .circleci/content-trust.key ~/.docker/trust/private/b056f84873aa0be205dfe826afa6e7458120c9569dd19a2a84154498fb1165d5.key @@ -293,39 +402,55 @@ workflows: - pkg-cri-containerd: requires: - dependencies - - pkg-kube-e2e-test: - requires: - - dependencies - - pkg-kubernetes-docker-image-cache-common: - requires: - - dependencies - - pkg-kubernetes-docker-image-cache-control-plane: - requires: - - dependencies + #- pkg-kube-e2e-test: + # requires: + # - dependencies + #- pkg-kubernetes-docker-image-cache-common: + # requires: + # - dependencies + #- pkg-kubernetes-docker-image-cache-control-plane: + # requires: + # - dependencies - - image-docker-weave: - requires: - - dependencies - - pkg-kubelet - - pkg-kubernetes-docker-image-cache-common - - pkg-kubernetes-docker-image-cache-control-plane - - image-docker-bridge: - requires: - - dependencies - - pkg-kubelet - - pkg-kubernetes-docker-image-cache-common - - pkg-kubernetes-docker-image-cache-control-plane - - image-cri-containerd-weave: - requires: - - dependencies - - pkg-kubelet - - pkg-cri-containerd + #- image-docker-weave: + # requires: + # - dependencies + # - pkg-kubelet + # - pkg-kubernetes-docker-image-cache-common + # - pkg-kubernetes-docker-image-cache-control-plane + #- image-docker-bridge: + # requires: + # - dependencies + # - pkg-kubelet + # - pkg-kubernetes-docker-image-cache-common + # - pkg-kubernetes-docker-image-cache-control-plane + #- image-cri-containerd-weave: + # requires: + # - dependencies + # - pkg-kubelet + # - pkg-cri-containerd - image-cri-containerd-bridge: requires: - dependencies - pkg-kubelet - pkg-cri-containerd + #- boot-docker-weave: + # requires: + # - image-docker-weave + + #- boot-docker-bridge: + # requires: + # - image-docker-bridge + + #- boot-cri-containerd-weave: + # requires: + # - image-cri-containerd-weave + + - boot-cri-containerd-bridge: + requires: + - image-cri-containerd-bridge + - push-pkgs-to-hub: # We want everything to have passed, which is a bit # tedious. Some of these are already covered transitively, @@ -334,10 +459,14 @@ workflows: - lint - pkg-kubelet - pkg-cri-containerd - - pkg-kube-e2e-test - - pkg-kubernetes-docker-image-cache-common - - pkg-kubernetes-docker-image-cache-control-plane - - image-docker-weave - - image-docker-bridge - - image-cri-containerd-weave + #- pkg-kube-e2e-test + #- pkg-kubernetes-docker-image-cache-common + #- pkg-kubernetes-docker-image-cache-control-plane + #- image-docker-weave + #- image-docker-bridge + #- image-cri-containerd-weave - image-cri-containerd-bridge + #- boot-docker-weave + #- boot-docker-bridge + #- boot-cri-containerd-weave + - boot-cri-containerd-bridge diff --git a/Makefile b/Makefile index 805a8ee..7c3c60e 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,8 @@ KUBE_FORMAT_ARGS := $(patsubst %,-format %,$(KUBE_FORMATS)) all: master node master: yml/kube.yml yml/$(KUBE_RUNTIME).yml yml/$(KUBE_RUNTIME)-master.yml yml/$(KUBE_NETWORK).yml + # strace logs received signals + #strace -f -e trace=signal linuxkit $(LINUXKIT_ARGS) build $(LINUXKIT_BUILD_ARGS) -name kube-master $(KUBE_FORMAT_ARGS) $^ node: yml/kube.yml yml/$(KUBE_RUNTIME).yml yml/$(KUBE_NETWORK).yml diff --git a/boot.sh b/boot.sh index a46b0a3..056c258 100755 --- a/boot.sh +++ b/boot.sh @@ -88,4 +88,4 @@ if [ -n "${kubeadm_data}" ] ; then echo "{ \"kubeadm\": { \"entries\": { ${kubeadm_data} } } }" > $state/metadata.json fi -exec linuxkit run ${KUBE_RUN_ARGS} -networking ${KUBE_NETWORKING} -cpus ${KUBE_VCPUS} -mem ${KUBE_MEM} -state "${state}" -disk size=${KUBE_DISK} -data $state/metadata.json ${uefi} "${img}${suffix}" +exec linuxkit -v run ${KUBE_RUN_ARGS} -networking ${KUBE_NETWORKING} -cpus ${KUBE_VCPUS} -mem ${KUBE_MEM} -state "${state}" -disk size=${KUBE_DISK} -data $state/metadata.json ${uefi} "${img}${suffix}" diff --git a/test.exp b/test.exp new file mode 100755 index 0000000..21da2e1 --- /dev/null +++ b/test.exp @@ -0,0 +1,199 @@ +#!/usr/bin/env expect + +set con_prompt "(ns: getty) linuxkit-*:*# " +set ssh_prompt "linuxkit-*:*# " +set timeout 120 + +proc kill args { + foreach what $args { + global $what + if [info exists $what] { + upvar #0 $what sid + set pid [exp_pid -i $sid] + puts "killing $what ($sid) = $pid" + exec kill $pid + close $sid + } else { + puts "not killing $what (not started)" + } + } +} + +proc boot_linuxkit {} { + global lk_sid + # "medium" level circle CI has 4G of RAM, use 3.5G for VM + # small 1.0 2GB + # medium (default) 2.0 4GB (3584M) + # medium+ 3.0 6GB + # large 4.0 8GB (7680M) + # xlarge 8.0 16GB (15827M) + spawn env {KUBE_RUN_ARGS=-publish 2222:22} KUBE_MEM=3584 KUBE_VCPUS=2 KUBE_DISK=8G KUBE_CLEAR_STATE=y KUBE_MASTER_UNTAINT=y ./boot.sh + set lk_sid $spawn_id + puts "INFO lk ($lk_sid) is pid [exp_pid -i $lk_sid]" +} + +proc ssh_into_kubelet {} { + global ssh_sid + + set sshopts {-p 2222 -o ConnectTimeout=5 -o LogLevel=DEBUG} + if [info exists ::env(SSHOPTS)] { + set sshopts [concat $::env(SSHOPTS) $sshopts] + } + spawn env SSHOPTS=$sshopts ./ssh_into_kubelet.sh localhost + set ssh_sid $spawn_id + puts "INFO ssh ($ssh_sid) is pid [exp_pid -i $ssh_sid]" +} + +proc await_prompt {sidvar promptvar step} { + upvar #0 $sidvar sid $promptvar prompt + expect -i $sid \ + $prompt { + puts "SUCCESS $step" + } timeout { + puts "FAIL $step (timeout)" + kill ssh_sid lk_sid + exit 1 + } eof { + puts "FAIL $step (eof)" + kill ssh_sid lk_sid + exit 1 + } +} + +proc send_con {s} { + global lk_sid + send -i $lk_sid $s +} + +proc await_con_prompt {step} { + global lk_sid con_prompt + await_prompt lk_sid con_prompt $step +} + +proc send_ssh {s} { + global ssh_sid + send -i $ssh_sid $s +} + +proc await_ssh_prompt {step} { + global ssh_sid ssh_prompt + await_prompt ssh_sid ssh_prompt $step +} + +boot_linuxkit + +await_con_prompt "boot" + +send_con "ifconfig eth0\n" +await_con_prompt "ifconfig" + +send_con "ctr container ls\n" +await_con_prompt "ctr container ls" + +send_con "ctr task ls\n" +# This was added in a debug attempt, but it turned out sshd was failing to come up (which this didn't help us to discover), so not much use now. +#set retries 0 +#expect -i lk_sid -timeout 300 \ +# "sshd * RUNNING" { +# puts "SUCCESS sshd running" +# } $con_prompt { +# set retries [expr $retries + 1] +# puts "RETRY:$retries await sshd" +# sleep 1 +# send_con "ctr task ls\n" +# exp_continue -continue_timer +# } timeout { +# puts "FAIL sshd (timeout)" +# kill ssh_sid lk_sid +# } +#await_con_prompt "post sshd startup (after $retries attempts)" + +set retries 0 +ssh_into_kubelet +# provide ssh_sid as an indirect, allowing ssh to be respawned, which +# changes the id, we need this in case ssh cannot immediately connect. +expect -i ssh_sid \ + $ssh_prompt { + puts "SUCCESS connected to ssh (after $retries attempts)" + } "read: Connection reset by peer" { + # ssh happened too soon, wait a bit. + set retries [expr $retries + 1] + puts "RETRY:$retries ssh (conn reset)" + wait -i $ssh_sid + sleep 1 + ssh_into_kubelet + exp_continue -continue_timer + } eof { + set retries [expr $retries + 1] + puts "RETRY:$retries ssh (eof)" + wait -i $ssh_sid + sleep 1 + ssh_into_kubelet + exp_continue -continue_timer + } timeout { + puts "FAIL ssh (timeout)" + kill ssh_sid lk_sid + exit 1 + } + +puts "RUN kubeadm-init.sh" +send_ssh "kubeadm-init.sh\n" + +# Written as N*5m with logging to avoid "10 mins with no output" from CI +set retries 0 +set maxretries 10 +expect -i $ssh_sid -timeout 300 \ + "Your Kubernetes master has initialized successfully!" { + puts "SUCCESS cluster initialised!" + } $ssh_prompt { + puts "FAIL kubeadm-init.sh (returned to prompt)" + kill ssh_sid lk_sid + exit 1 + } timeout { + set retries [expr $retries + 1] + if [expr $retries < $maxretries] { + puts "RETRY:$retries kubeadm-init.sh (timeout)" + exp_continue + } + puts "FAIL kubeadm-init.sh (timeout)" + kill ssh_sid lk_sid + exit 1 + } eof { + puts "FAIL kubeadm-init.sh (eof)" + kill ssh_sid lk_sid + exit 1 + } +await_ssh_prompt "kubeadm-init.sh" +puts "SUCCESS kubeadm-init complete" + +send_ssh "kubectl get nodes\n" +set retries 0 +expect -i lk_sid -timeout 300 \ + "linuxkit-* Ready" { + puts "SUCCESS node ready" + } $ssh_prompt { + set retries [expr $retries + 1] + puts "RETRY:$retries await node ready" + sleep 1 + send_con "kubectl get nodes\n" + exp_continue -continue_timer + } timeout { + puts "FAIL node ready (timeout)" + kill ssh_sid lk_sid + } +await_ssh_prompt "post await node ready (after $retries attempts)" + +kill ssh_sid + +puts "RUN poweroff -f" +send_con "poweroff -f\n" + +expect -i $lk_sid \ + "Power down" { + puts "SUCCESS poweroff" + } eof { + puts "SUCCESS poweroff" + } timeout { + puts "FAILED poweroff (timeout)" + exit 1 + }