Deferred creation of SkyDNS, monitoring and logging objects

This implements phase 1 of the proposal in #3579, moving the creation
of the pods, RCs, and services to the master after the apiserver is
available.

This is such a wide commit because our existing initial config story
is special:

* Add kube-addons service and associated salt configuration:
** We configure /etc/kubernetes/addons to be a directory of objects
that are appropriately configured for the current cluster.
** "/etc/init.d/kube-addons start" slurps up everything in that dir.
(Most of the difficult is the business logic in salt around getting
that directory built at all.)
** We cheat and overlay cluster/addons into saltbase/salt/kube-addons
as config files for the kube-addons meta-service.
* Change .yaml.in files to salt templates
* Rename {setup,teardown}-{monitoring,logging} to
{setup,teardown}-{monitoring,logging}-firewall to properly reflect
their real purpose now (the purpose of these functions is now ONLY to
bring up the firewall rules, and possibly to relay the IP to the user).
* Rework GCE {setup,teardown}-{monitoring,logging}-firewall: Both
functions were improperly configuring global rules, yet used
lifecycles tied to the cluster. Use $NODE_INSTANCE_PREFIX with the
rule. The logging rule needed a $NETWORK specifier. The monitoring
rule tried gcloud describe first, but given the instancing, this feels
like a waste of time now.
* Plumb ENABLE_CLUSTER_MONITORING, ENABLE_CLUSTER_LOGGING,
ELASTICSEARCH_LOGGING_REPLICAS and DNS_REPLICAS down to the master,
since these are needed there now.

(Desperately want just a yaml or json file we can share between
providers that has all this crap. Maybe #3525 is an answer?)

Huge caveats: I've gone pretty firm testing on GCE, including
twiddling the env variables and making sure the objects I expect to
come up, come up. I've tested that it doesn't break GKE bringup
somehow. But I haven't had a chance to test the other providers.

init.sls

@@ -0,0 +1,79 @@
+{% if pillar.get('enable_cluster_monitoring', '').lower() == 'true' %}
+/etc/kubernetes/addons/cluster-monitoring:
+  file.recurse:
+    - source: salt://kube-addons/cluster-monitoring
+    - include_pat: E@^.+\.yaml$
+    - user: root
+    - group: root
+    - dir_mode: 755
+    - file_mode: 644
+{% endif %}
+
+{% if pillar.get('enable_cluster_dns', '').lower() == 'true' %}
+/etc/kubernetes/addons/dns/skydns-svc.yaml:
+  file.managed:
+    - source: salt://kube-addons/dns/skydns-svc.yaml.in
+    - template: jinja
+    - group: root
+    - dir_mode: 755
+    - makedirs: True
+
+/etc/kubernetes/addons/dns/skydns-rc.yaml:
+  file.managed:
+    - source: salt://kube-addons/dns/skydns-rc.yaml.in
+    - template: jinja
+    - group: root
+    - dir_mode: 755
+    - makedirs: True
+{% endif %}
+
+{% if pillar.get('enable_node_logging', '').lower() == 'true'
+   and pillar.get('logging_destination').lower() == 'elasticsearch'
+   and pillar.get('enable_cluster_logging', '').lower() == 'true' %}
+/etc/kubernetes/addons/fluentd-elasticsearch:
+  file.recurse:
+    - source: salt://kube-addons/fluentd-elasticsearch
+    - include_pat: E@^.+\.yaml$
+    - user: root
+    - group: root
+    - dir_mode: 755
+    - file_mode: 644
+
+/etc/kubernetes/addons/fluentd-elasticsearch/es-controller.yaml:
+  file.managed:
+    - source: salt://kube-addons/fluentd-elasticsearch/es-controller.yaml.in
+    - template: jinja
+    - group: root
+    - dir_mode: 755
+    - makedirs: True
+{% endif %}
+
+{% if grains['os_family'] == 'RedHat' %}
+
+/usr/lib/systemd/system/kube-addons.service:
+  file.managed:
+    - source: salt://kube-addons/kube-addons.service
+    - user: root
+    - group: root
+
+/usr/lib/systemd/scripts/kube-addons:
+  file.managed:
+    - source: salt://kube-addons/initd
+    - user: root
+    - group: root
+    - mode: 755
+
+{% else %}
+
+/etc/init.d/kube-addons:
+  file.managed:
+    - source: salt://kube-addons/initd
+    - user: root
+    - group: root
+    - mode: 755
+
+{% endif %}
+
+kube-addons:
+  service.running:
+    - enable: True

initd

@@ -0,0 +1,123 @@
+#!/bin/bash
+#
+### BEGIN INIT INFO
+# Provides:   kube-addons
+# Required-Start:    $local_fs $network $syslog kube-apiserver
+# Required-Stop:
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Kubernetes Addon Object Manager
+# Description:
+#   Enforces installation of Kubernetes Addon Objects
+### END INIT INFO
+
+
+# PATH should only include /usr/* if it runs after the mountnfs.sh script
+PATH=/sbin:/usr/sbin:/bin:/usr/bin
+DESC="Kubernetes Addon Object Manager"
+NAME=kube-addons
+DAEMON_LOG_FILE=/var/log/$NAME.log
+PIDFILE=/var/run/$NAME.pid
+SCRIPTNAME=/etc/init.d/$NAME
+KUBECTL=/usr/local/bin/kubectl
+
+# Define LSB log_* functions.
+# Depend on lsb-base (>= 3.2-14) to ensure that this file is present
+# and status_of_proc is working.
+. /lib/lsb/init-functions
+
+function addon_manager_async() {
+    # The business logic for whether a given object should be created
+    # was already enforced by salt, and /etc/kubernetes/addons is the
+    # managed result is of that. Start everything below that directory.
+    echo "== Kubernetes addon manager started at $(date -Is) =="
+    for obj in $(find /etc/kubernetes/addons -name \*.yaml); do
+	    ${KUBECTL} create -f ${obj} &
+	    echo "++ addon ${obj} started in pid $! ++"
+    done
+    noerrors="true"
+    for pid in $(jobs -p); do
+	    wait ${pid} || noerrors="false"
+	    echo "++ pid ${pid} complete ++"
+    done
+    if [ ${noerrors} == "true" ]; then
+	    echo "== Kubernetes addon manager completed successfully at $(date -Is) =="
+    else
+	    echo "== Kubernetes addon manager completed with errors at $(date -Is) =="
+    fi
+
+    # We stay around so that status checks by salt make it look like
+    # the service is good. (We could do this is other ways, but this
+    # is simple.)
+    sleep infinity
+}
+
+#
+# Function that starts the daemon/service
+#
+do_start()
+{
+    addon_manager_async </dev/null >>${DAEMON_LOG_FILE} 2>&1 &
+    echo $! > ${PIDFILE}
+    disown
+}
+
+#
+# Function that stops the daemon/service
+#
+do_stop()
+{
+    kill $(cat ${PIDFILE})
+    rm ${PIDFILE}
+    return
+}
+
+case "$1" in
+  start)
+        log_daemon_msg "Starting $DESC" "$NAME"
+        do_start
+        case "$?" in
+                0|1) log_end_msg 0 || exit 0 ;;
+                2) log_end_msg 1 || exit 1 ;;
+        esac
+        ;;
+  stop)
+        log_daemon_msg "Stopping $DESC" "$NAME"
+        do_stop
+        case "$?" in
+                0|1) log_end_msg 0 ;;
+                2) exit 1 ;;
+        esac
+        ;;
+  status)
+	    if [ ! -e ${PIDFILE} ]; then
+	        exit 1
+	    fi
+	    pid=$(cat ${PIDFILE})
+        # Checks that ${pid} is running AND is us.
+        ps --no-headers ${pid} | grep ${SCRIPTNAME} > /dev/null || exit $?
+        ;;
+
+  restart|force-reload)
+        log_daemon_msg "Restarting $DESC" "$NAME"
+        do_stop
+        case "$?" in
+          0|1)
+                do_start
+                case "$?" in
+                        0) log_end_msg 0 ;;
+                        1) log_end_msg 1 ;; # Old process is still running
+                        *) log_end_msg 1 ;; # Failed to start
+                esac
+                ;;
+          *)
+                # Failed to stop
+                log_end_msg 1
+                ;;
+        esac
+        ;;
+  *)
+        echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2
+        exit 3
+        ;;
+esac

kube-addons.service

@@ -0,0 +1,9 @@
+[Unit]
+Description=Kubernetes Addon Object Manager
+Documentation=https://github.com/GoogleCloudPlatform/kubernetes
+
+[Service]
+ExecStart=/usr/lib/systemd/scripts/kube-addons start
+
+[Install]
+WantedBy=multi-user.target