Compress ACME certificates in KV stores.

This commit is contained in:
NicoMen 2018-02-09 10:38:03 +01:00 committed by Traefiker
parent f07e8f58e6
commit 2b1d2853cd
11 changed files with 182 additions and 27 deletions

6
Gopkg.lock generated
View file

@ -223,8 +223,8 @@
[[projects]] [[projects]]
name = "github.com/containous/staert" name = "github.com/containous/staert"
packages = ["."] packages = ["."]
revision = "af517d5b70db9c4b0505e0144fcc62b054057d2a" revision = "68c67b32c3a986672d994d38127cd5c78d53eb26"
version = "v2.0.0" version = "v2.1.0"
[[projects]] [[projects]]
name = "github.com/containous/traefik-extra-service-fabric" name = "github.com/containous/traefik-extra-service-fabric"
@ -1387,6 +1387,6 @@
[solve-meta] [solve-meta]
analyzer-name = "dep" analyzer-name = "dep"
analyzer-version = 1 analyzer-version = 1
inputs-digest = "691b593c653c992681b518be011637c40d978c9366f518ced19a442ab2acd6d7" inputs-digest = "bd1e7a1b07d95ff85c675468bbfc4bc7a91c39cf1feceeb58dfcdba9592180a5"
solver-name = "gps-cdcl" solver-name = "gps-cdcl"
solver-version = 1 solver-version = 1

View file

@ -64,7 +64,7 @@ ignored = ["github.com/sirupsen/logrus"]
[[constraint]] [[constraint]]
name = "github.com/containous/staert" name = "github.com/containous/staert"
version = "2.0.0" version = "2.1.0"
[[constraint]] [[constraint]]
name = "github.com/containous/traefik-extra-service-fabric" name = "github.com/containous/traefik-extra-service-fabric"

View file

@ -165,9 +165,26 @@ storage = "acme.json"
# ... # ...
``` ```
File or key used for certificates storage. The `storage` option sets where are stored your ACME certificates.
**WARNING:** If you use Træfik in Docker, you have 2 options: There are two kind of `storage` :
- a JSON file,
- a KV store entry.
!!! danger "DEPRECATED"
`storage` replaces `storageFile` which is deprecated.
!!! note
During Træfik configuration migration from a configuration file to a KV store (thanks to `storeconfig` subcommand as described [here](/user-guide/kv-config/#store-configuration-in-key-value-store)), if ACME certificates have to be migrated too, use both `storageFile` and `storage`.
- `storageFile` will contain the path to the `acme.json` file to migrate.
- `storage` will contain the key where the certificates will be stored.
#### Store data in a file
ACME certificates can be stored in a JSON file which with the `600` right mode.
There are two ways to store ACME certificates in a file from Docker:
- create a file on your host and mount it as a volume: - create a file on your host and mount it as a volume:
```toml ```toml
@ -176,7 +193,6 @@ storage = "acme.json"
```bash ```bash
docker run -v "/my/host/acme.json:acme.json" traefik docker run -v "/my/host/acme.json:acme.json" traefik
``` ```
- mount the folder containing the file as a volume - mount the folder containing the file as a volume
```toml ```toml
storage = "/etc/traefik/acme/acme.json" storage = "/etc/traefik/acme/acme.json"
@ -185,14 +201,24 @@ storage = "/etc/traefik/acme/acme.json"
docker run -v "/my/host/acme:/etc/traefik/acme" traefik docker run -v "/my/host/acme:/etc/traefik/acme" traefik
``` ```
!!! note !!! warning
`storage` replaces `storageFile` which is deprecated. This file cannot be shared per many instances of Træfik at the same time.
If you have to use Træfik cluster mode, please use [a KV Store entry](/configuration/acme/#storage-kv-entry).
#### Store data in a KV store entry
ACME certificates can be stored in a KV Store entry.
```toml
storage = "traefik/acme/account"
```
**This kind of storage is mandatory in cluster mode.**
Because KV stores (like Consul) have limited entries size, the certificates list is compressed before to be set in a KV store entry.
!!! note !!! note
During Træfik configuration migration from a configuration file to a KV store (thanks to `storeconfig` subcommand as described [here](/user-guide/kv-config/#store-configuration-in-key-value-store)), if ACME certificates have to be migrated too, use both `storageFile` and `storage`. It's possible to store up to approximately 100 ACME certificates in Consul.
- `storageFile` will contain the path to the `acme.json` file to migrate.
- `storage` will contain the key where the certificates will be stored.
### `acme.httpChallenge` ### `acme.httpChallenge`
@ -288,7 +314,7 @@ Useful if internal networks block external DNS queries.
### `onDemand` (Deprecated) ### `onDemand` (Deprecated)
!!! warning !!! danger "DEPRECATED"
This option is deprecated. This option is deprecated.
```toml ```toml
@ -365,12 +391,12 @@ Each domain & SANs will lead to a certificate request.
### `dnsProvider` (Deprecated) ### `dnsProvider` (Deprecated)
!!! warning !!! danger "DEPRECATED"
This option is deprecated. This option is deprecated.
Please refer to [DNS challenge provider section](/configuration/acme/#provider) Please refer to [DNS challenge provider section](/configuration/acme/#provider)
### `delayDontCheckDNS` (Deprecated) ### `delayDontCheckDNS` (Deprecated)
!!! warning !!! danger "DEPRECATED"
This option is deprecated. This option is deprecated.
Please refer to [DNS challenge delayBeforeCheck section](/configuration/acme/#delaybeforecheck) Please refer to [DNS challenge delayBeforeCheck section](/configuration/acme/#delaybeforecheck)

View file

@ -211,9 +211,8 @@ In the example below both `snitest.com` and `snitest.org` will require client ce
``` ```
!!! note !!! note
The deprecated argument `ClientCAFiles` allows adding Client CA files which are mandatory.
The deprecated argument `ClientCAFiles` allows adding Client CA files which are mandatory. If this parameter exists, the new ones are not checked.
If this parameter exists, the new ones are not checked.
## Authentication ## Authentication

View file

@ -23,3 +23,11 @@ A Træfik cluster is based on a manager/worker model.
When starting, Træfik will elect a manager. When starting, Træfik will elect a manager.
If this instance fails, another manager will be automatically elected. If this instance fails, another manager will be automatically elected.
## Træfik cluster and Let's Encrypt
**In cluster mode, ACME certificates have to be stored in [a KV Store entry](/configuration/acme/#storage-kv-entry).**
Thanks to the Træfik cluster mode algorithm (based on [the Raft Consensus Algorithm](https://raft.github.io/)), only one instance will contact Let's encrypt to solve the challenges.
The others instances will get ACME certificate from the KV Store entry.

View file

@ -29,6 +29,8 @@ services :
- bhsm - bhsm
- bmysql - bmysql
- brabbitmq - brabbitmq
volumes:
- "./rate-limit-policies.yml:/go/src/github.com/letsencrypt/boulder/test/rate-limit-policies.yml:ro"
bhsm: bhsm:
image: letsencrypt/boulder-tools:2016-11-02 image: letsencrypt/boulder-tools:2016-11-02

View file

@ -0,0 +1,42 @@
totalCertificates:
window: 1h
threshold: 100000
certificatesPerName:
window: 1h
threshold: 100000
overrides:
ratelimit.me: 1
lim.it: 0
# Hostnames used by the letsencrypt client integration test.
le.wtf: 10000
le1.wtf: 10000
le2.wtf: 10000
le3.wtf: 10000
nginx.wtf: 10000
good-caa-reserved.com: 10000
bad-caa-reserved.com: 10000
ecdsa.le.wtf: 10000
must-staple.le.wtf: 10000
registrationOverrides:
101: 1000
registrationsPerIP:
window: 1h
threshold: 100000
overrides:
127.0.0.1: 1000000
pendingAuthorizationsPerAccount:
window: 1h
threshold: 100000
certificatesPerFQDNSet:
window: 1h
threshold: 100000
overrides:
le.wtf: 10000
le1.wtf: 10000
le2.wtf: 10000
le3.wtf: 10000
le.wtf,le1.wtf: 10000
good-caa-reserved.com: 10000
nginx.wtf: 10000
ecdsa.le.wtf: 10000
must-staple.le.wtf: 10000

View file

@ -73,6 +73,8 @@ services:
- bhsm - bhsm
- bmysql - bmysql
- brabbitmq - brabbitmq
volumes:
- "./rate-limit-policies.yml:/go/src/github.com/letsencrypt/boulder/test/rate-limit-policies.yml:ro"
networks: networks:
net: net:
ipv4_address: 10.0.1.3 ipv4_address: 10.0.1.3

View file

@ -0,0 +1,42 @@
totalCertificates:
window: 1h
threshold: 100000
certificatesPerName:
window: 1h
threshold: 100000
overrides:
ratelimit.me: 1
lim.it: 0
# Hostnames used by the letsencrypt client integration test.
le.wtf: 10000
le1.wtf: 10000
le2.wtf: 10000
le3.wtf: 10000
nginx.wtf: 10000
good-caa-reserved.com: 10000
bad-caa-reserved.com: 10000
ecdsa.le.wtf: 10000
must-staple.le.wtf: 10000
registrationOverrides:
101: 1000
registrationsPerIP:
window: 1h
threshold: 100000
overrides:
127.0.0.1: 1000000
pendingAuthorizationsPerAccount:
window: 1h
threshold: 100000
certificatesPerFQDNSet:
window: 1h
threshold: 100000
overrides:
le.wtf: 10000
le1.wtf: 10000
le2.wtf: 10000
le3.wtf: 10000
le.wtf,le1.wtf: 10000
good-caa-reserved.com: 10000
nginx.wtf: 10000
ecdsa.le.wtf: 10000
must-staple.le.wtf: 10000

View file

@ -19,8 +19,7 @@ caServer = "http://traefik.boulder.com:4000/directory"
entryPoint="http" entryPoint="http"
[web] [api]
address = ":8080"
[docker] [docker]
endpoint = "unix:///var/run/docker.sock" endpoint = "unix:///var/run/docker.sock"

View file

@ -1,10 +1,14 @@
package staert package staert
import ( import (
"bytes"
"compress/gzip"
"encoding" "encoding"
"encoding/base64" "encoding/base64"
"errors" "errors"
"fmt" "fmt"
"io"
"io/ioutil"
"reflect" "reflect"
"sort" "sort"
"strconv" "strconv"
@ -155,16 +159,32 @@ func decodeHook(fromType reflect.Type, toType reflect.Type, data interface{}) (i
return dataOutput, nil return dataOutput, nil
} else if fromType.Kind() == reflect.String { } else if fromType.Kind() == reflect.String {
b, err := base64.StdEncoding.DecodeString(data.(string)) return readCompressedData(data.(string), gzipReader, base64Reader)
if err != nil {
return nil, err
}
return b, nil
} }
} }
return data, nil return data, nil
} }
func readCompressedData(data string, fs ...func(io.Reader) (io.Reader, error)) ([]byte, error) {
var err error
for _, f := range fs {
var reader io.Reader
reader, err = f(bytes.NewBufferString(data))
if err == nil {
return ioutil.ReadAll(reader)
}
}
return nil, err
}
func base64Reader(r io.Reader) (io.Reader, error) {
return base64.NewDecoder(base64.StdEncoding, r), nil
}
func gzipReader(r io.Reader) (io.Reader, error) {
return gzip.NewReader(r)
}
// StoreConfig stores the config into the KV Store // StoreConfig stores the config into the KV Store
func (kv *KvSource) StoreConfig(config interface{}) error { func (kv *KvSource) StoreConfig(config interface{}) error {
kvMap := map[string]string{} kvMap := map[string]string{}
@ -263,7 +283,11 @@ func collateKvRecursive(objValue reflect.Value, kv map[string]string, key string
case reflect.Array, reflect.Slice: case reflect.Array, reflect.Slice:
// Byte slices get special treatment // Byte slices get special treatment
if objValue.Type().Elem().Kind() == reflect.Uint8 { if objValue.Type().Elem().Kind() == reflect.Uint8 {
kv[name] = base64.StdEncoding.EncodeToString(objValue.Bytes()) compressedData, err := writeCompressedData(objValue.Bytes())
if err != nil {
return err
}
kv[name] = compressedData
} else { } else {
for i := 0; i < objValue.Len(); i++ { for i := 0; i < objValue.Len(); i++ {
name = key + "/" + strconv.Itoa(i) name = key + "/" + strconv.Itoa(i)
@ -286,6 +310,17 @@ func collateKvRecursive(objValue reflect.Value, kv map[string]string, key string
return nil return nil
} }
func writeCompressedData(data []byte) (string, error) {
var buffer bytes.Buffer
gzipWriter := gzip.NewWriter(&buffer)
_, err := gzipWriter.Write(data)
if err != nil {
return "", err
}
gzipWriter.Close()
return buffer.String(), nil
}
// ListRecursive lists all key value children under key // ListRecursive lists all key value children under key
func (kv *KvSource) ListRecursive(key string, pairs map[string][]byte) error { func (kv *KvSource) ListRecursive(key string, pairs map[string][]byte) error {
pairsN1, err := kv.List(key, nil) pairsN1, err := kv.List(key, nil)