Files
cairnobs/internal/kube/kube.go
T
jcoffey-dev db156c4f06
release / release (push) Successful in 7m6s
Cairn OBS
The new Cairn OBS: a self-hosted monitor for networks, hosts and
services. Licensed AGPLv3.
2026-11-14 03:27:09 -07:00

318 lines
8.9 KiB
Go

// Package kube is the little of the Kubernetes API the cluster agent uses:
// get, list (in pages) and watch, plus patching its own Secret. It speaks
// JSON over HTTPS with the pod's service account, and decodes only what
// the caller asks for, so whole objects never sit in memory.
//
// It's not client-go: that would be most of the agent's size for four
// calls (docs/cluster-watcher.md).
package kube
import (
"bufio"
"bytes"
"context"
"crypto/tls"
"crypto/x509"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
)
// The service account's files in every pod that mounts its token.
const saDir = "/var/run/secrets/kubernetes.io/serviceaccount"
// pageSize is how many objects a list asks for at a time.
const pageSize = 500
// Client talks to one cluster's API.
type Client struct {
base string
http *http.Client // for get, list and patch: with a timeout
stream *http.Client // for watches: no overall timeout, the context ends them
tokenFile string
namespace string
mu sync.Mutex
token string
tokenRead time.Time
}
// InCluster is the client for the cluster the agent runs in, from the
// service account Kubernetes mounts.
func InCluster() (*Client, error) {
host, port := os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT")
if host == "" || port == "" {
return nil, errors.New("not in a cluster: KUBERNETES_SERVICE_HOST and _PORT aren't set")
}
ca, err := os.ReadFile(filepath.Join(saDir, "ca.crt"))
if err != nil {
return nil, err
}
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(ca) {
return nil, errors.New("the service account's ca.crt holds no certificate")
}
ns, err := os.ReadFile(filepath.Join(saDir, "namespace"))
if err != nil {
return nil, err
}
c := New("https://"+net.JoinHostPort(host, port), &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, filepath.Join(saDir, "token"))
c.namespace = strings.TrimSpace(string(ns))
if v := os.Getenv("POD_NAMESPACE"); v != "" {
c.namespace = v
}
return c, nil
}
// New is a client for base (https://host:port) with tc, authenticating
// with the bearer token in tokenFile ("" for none). Tests use it with an
// httptest server.
func New(base string, tc *tls.Config, tokenFile string) *Client {
tr := func() *http.Transport {
return &http.Transport{TLSClientConfig: tc, ForceAttemptHTTP2: true, MaxIdleConns: 4, IdleConnTimeout: time.Minute,
ResponseHeaderTimeout: 30 * time.Second}
}
return &Client{base: strings.TrimRight(base, "/"), tokenFile: tokenFile,
http: &http.Client{Timeout: time.Minute, Transport: tr()},
stream: &http.Client{Transport: tr()}}
}
// Namespace is the agent's own namespace.
func (c *Client) Namespace() string { return c.namespace }
// bearer is the service account token, read again every minute: the
// kubelet replaces it well before it runs out.
func (c *Client) bearer() (string, error) {
if c.tokenFile == "" {
return "", nil
}
c.mu.Lock()
defer c.mu.Unlock()
if c.token != "" && time.Since(c.tokenRead) < time.Minute {
return c.token, nil
}
b, err := os.ReadFile(c.tokenFile)
if err != nil {
if c.token != "" {
return c.token, nil // keep the last one; it's good for a while
}
return "", err
}
c.token, c.tokenRead = strings.TrimSpace(string(b)), time.Now()
return c.token, nil
}
// StatusError is an answer other than 2xx: the API's Status, when it sent
// one.
type StatusError struct {
Code int
Reason string // Forbidden, NotFound, Expired...
Message string
}
func (e *StatusError) Error() string {
if e.Message != "" {
return fmt.Sprintf("%d %s", e.Code, e.Message)
}
return fmt.Sprintf("%d %s", e.Code, http.StatusText(e.Code))
}
// Code is the HTTP status of err, 0 when it isn't a StatusError.
func Code(err error) int {
var se *StatusError
if errors.As(err, &se) {
return se.Code
}
return 0
}
// ErrGone is a watch or list from a resourceVersion the API no longer
// has: list again.
var ErrGone = errors.New("resource version too old")
func statusError(code int, body []byte) error {
var st struct {
Reason string `json:"reason"`
Message string `json:"message"`
Code int `json:"code"`
}
_ = json.Unmarshal(body, &st)
if code == http.StatusGone || st.Code == http.StatusGone {
return ErrGone
}
if st.Message == "" {
st.Message = strings.TrimSpace(string(body))
if len(st.Message) > 200 {
st.Message = st.Message[:200]
}
}
return &StatusError{Code: code, Reason: st.Reason, Message: st.Message}
}
func (c *Client) do(ctx context.Context, hc *http.Client, method, path string, q url.Values, body []byte, ctype string) (*http.Response, error) {
u := c.base + path
if len(q) > 0 {
u += "?" + q.Encode()
}
var rd io.Reader
if body != nil {
rd = bytes.NewReader(body)
}
req, err := http.NewRequestWithContext(ctx, method, u, rd)
if err != nil {
return nil, err
}
tok, err := c.bearer()
if err != nil {
return nil, err
}
if tok != "" {
req.Header.Set("Authorization", "Bearer "+tok)
}
req.Header.Set("Accept", "application/json")
if ctype != "" {
req.Header.Set("Content-Type", ctype)
}
resp, err := hc.Do(req)
if err != nil {
return nil, err
}
if resp.StatusCode/100 != 2 {
b, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
resp.Body.Close()
return nil, statusError(resp.StatusCode, b)
}
return resp, nil
}
// Get decodes the object at path into v.
func (c *Client) Get(ctx context.Context, path string, v any) error {
resp, err := c.do(ctx, c.http, http.MethodGet, path, nil, nil, "")
if err != nil {
return err
}
defer resp.Body.Close()
return json.NewDecoder(io.LimitReader(resp.Body, 8<<20)).Decode(v)
}
// MergePatch sends a JSON merge patch to the object at path.
func (c *Client) MergePatch(ctx context.Context, path string, patch any) error {
b, err := json.Marshal(patch)
if err != nil {
return err
}
resp, err := c.do(ctx, c.http, http.MethodPatch, path, nil, b, "application/merge-patch+json")
if err != nil {
return err
}
_, _ = io.Copy(io.Discard, resp.Body)
return resp.Body.Close()
}
// List calls each with every object at path (a collection, such as
// /api/v1/pods), asking for them in pages, and returns the list's
// resourceVersion to watch from.
func (c *Client) List(ctx context.Context, path string, each func(raw json.RawMessage) error) (string, error) {
q := url.Values{"limit": {strconv.Itoa(pageSize)}}
for {
resp, err := c.do(ctx, c.http, http.MethodGet, path, q, nil, "")
if err != nil {
return "", err
}
var page struct {
Metadata struct {
ResourceVersion string `json:"resourceVersion"`
Continue string `json:"continue"`
} `json:"metadata"`
Items []json.RawMessage `json:"items"`
}
err = json.NewDecoder(resp.Body).Decode(&page)
resp.Body.Close()
if err != nil {
return "", fmt.Errorf("listing %s: %w", path, err)
}
for _, it := range page.Items {
if err := each(it); err != nil {
return "", err
}
}
if page.Metadata.Continue == "" {
return page.Metadata.ResourceVersion, nil
}
q.Set("continue", page.Metadata.Continue)
}
}
// Event types in a watch.
const (
Added = "ADDED"
Modified = "MODIFIED"
Deleted = "DELETED"
Bookmark = "BOOKMARK"
)
// Watch calls fn with each change at path from resourceVersion rv, until
// the API ends the watch (after timeout, or sooner), the context ends, or
// fn fails. It returns the last resourceVersion seen, bookmarks included,
// to watch on from; ErrGone means list again. Bookmarks don't reach fn.
func (c *Client) Watch(ctx context.Context, path, rv string, timeout time.Duration, fn func(typ string, raw json.RawMessage) error) (string, error) {
q := url.Values{"watch": {"1"}, "resourceVersion": {rv}, "allowWatchBookmarks": {"true"},
"timeoutSeconds": {strconv.Itoa(int(timeout.Seconds()))}}
resp, err := c.do(ctx, c.stream, http.MethodGet, path, q, nil, "")
if err != nil {
return rv, err
}
defer resp.Body.Close()
dec := json.NewDecoder(bufio.NewReaderSize(resp.Body, 64<<10))
for {
var ev struct {
Type string `json:"type"`
Object json.RawMessage `json:"object"`
}
if err := dec.Decode(&ev); err != nil {
if errors.Is(err, io.EOF) || ctx.Err() != nil {
return rv, ctx.Err()
}
return rv, err
}
if ev.Type == "ERROR" {
return rv, statusError(0, ev.Object)
}
var meta struct {
Metadata struct {
ResourceVersion string `json:"resourceVersion"`
} `json:"metadata"`
}
if err := json.Unmarshal(ev.Object, &meta); err == nil && meta.Metadata.ResourceVersion != "" {
rv = meta.Metadata.ResourceVersion
}
if ev.Type == Bookmark {
continue
}
if err := fn(ev.Type, ev.Object); err != nil {
return rv, err
}
}
}
// Version is the API server's version (/version), e.g. "v1.32.2".
func (c *Client) Version(ctx context.Context) (string, error) {
var v struct {
GitVersion string `json:"gitVersion"`
}
if err := c.Get(ctx, "/version", &v); err != nil {
return "", err
}
return v.GitVersion, nil
}