mirror of
https://github.com/distribution/distribution
synced 2024-09-06 02:48:53 +00:00
Merge pull request #381 from RichardScothern/purge-config
Add configuration for upload purging
This commit is contained in:
commit
70427e3d9f
5 changed files with 125 additions and 13 deletions
|
@ -9,6 +9,9 @@ storage:
|
|||
layerinfo: inmemory
|
||||
filesystem:
|
||||
rootdirectory: /tmp/registry-dev
|
||||
maintenance:
|
||||
uploadpurging:
|
||||
enabled: false
|
||||
http:
|
||||
addr: :5000
|
||||
secret: asecretforlocaldevelopment
|
||||
|
@ -39,3 +42,4 @@ notifications:
|
|||
threshold: 10
|
||||
backoff: 1s
|
||||
disabled: true
|
||||
|
|
@ -188,6 +188,8 @@ func (storage Storage) Type() string {
|
|||
// Return only key in this map
|
||||
for k := range storage {
|
||||
switch k {
|
||||
case "maintenance":
|
||||
// allow configuration of maintenance
|
||||
case "cache":
|
||||
// allow configuration of caching
|
||||
default:
|
||||
|
@ -217,6 +219,8 @@ func (storage *Storage) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
|||
types := make([]string, 0, len(storageMap))
|
||||
for k := range storageMap {
|
||||
switch k {
|
||||
case "maintenance":
|
||||
// allow for configuration of maintenance
|
||||
case "cache":
|
||||
// allow configuration of caching
|
||||
default:
|
||||
|
|
|
@ -55,6 +55,12 @@ storage:
|
|||
rootdirectory: /s3/object/name/prefix
|
||||
cache:
|
||||
layerinfo: inmemory
|
||||
maintenance:
|
||||
uploadpurging:
|
||||
enabled: true
|
||||
age: 168h
|
||||
interval: 24h
|
||||
dryrun: false
|
||||
auth:
|
||||
silly:
|
||||
realm: silly-realm
|
||||
|
@ -233,6 +239,12 @@ storage:
|
|||
rootdirectory: /s3/object/name/prefix
|
||||
cache:
|
||||
layerinfo: inmemory
|
||||
maintenance:
|
||||
uploadpurging:
|
||||
enabled: true
|
||||
age: 168h
|
||||
interval: 24h
|
||||
dryrun: false
|
||||
```
|
||||
|
||||
The storage option is **required** and defines which storage backend is in use.
|
||||
|
@ -422,6 +434,27 @@ This storage backend uses Amazon's Simple Storage Service (S3).
|
|||
</tr>
|
||||
</table>
|
||||
|
||||
### Maintenance
|
||||
|
||||
Currently the registry can perform one maintenance function: upload purging. This and future
|
||||
maintenance functions which are related to storage can be configured under the maintenance section.
|
||||
|
||||
### Upload Purging
|
||||
|
||||
Upload purging is a background process that periodically removes orphaned files from the upload
|
||||
directories of the registry. Upload purging is enabled by default. To
|
||||
configure upload directory purging, the following parameters
|
||||
must be set.
|
||||
|
||||
|
||||
| Parameter | Required | Description
|
||||
--------- | -------- | -----------
|
||||
`enabled` | yes | Set to true to enable upload purging. Default=true. |
|
||||
`age` | yes | Upload directories which are older than this age will be deleted. Default=168h (1 week)
|
||||
`interval` | yes | The interval between upload directory purging. Default=24h.
|
||||
`dryrun` | yes | dryrun can be set to true to obtain a summary of what directories will be deleted. Default=false.
|
||||
|
||||
Note: `age` and `interval` are strings containing a number with optional fraction and a unit suffix: e.g. 45m, 2h10m, 168h (1 week).
|
||||
|
||||
## auth
|
||||
|
||||
|
@ -1152,6 +1185,7 @@ Configure the behavior of the Redis connection pool.
|
|||
</tr>
|
||||
</table>
|
||||
|
||||
|
||||
## Example: Development configuration
|
||||
|
||||
The following is a simple example you can use for local development:
|
||||
|
|
|
@ -205,6 +205,9 @@ storage:
|
|||
layerinfo: inmemory
|
||||
filesystem:
|
||||
rootdirectory: /tmp/registry-dev
|
||||
maintenance:
|
||||
uploadpurging:
|
||||
enabled: false
|
||||
http:
|
||||
addr: :5000
|
||||
secret: asecretforlocaldevelopment
|
||||
|
|
|
@ -81,7 +81,18 @@ func NewApp(ctx context.Context, configuration configuration.Configuration) *App
|
|||
panic(err)
|
||||
}
|
||||
|
||||
startUploadPurger(app.driver, ctxu.GetLogger(app))
|
||||
purgeConfig := uploadPurgeDefaultConfig()
|
||||
if mc, ok := configuration.Storage["maintenance"]; ok {
|
||||
for k, v := range mc {
|
||||
switch k {
|
||||
case "uploadpurging":
|
||||
purgeConfig = v.(map[interface{}]interface{})
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
startUploadPurger(app.driver, ctxu.GetLogger(app), purgeConfig)
|
||||
|
||||
app.driver, err = applyStorageMiddleware(app.driver, configuration.Middleware["storage"])
|
||||
if err != nil {
|
||||
|
@ -568,26 +579,82 @@ func applyStorageMiddleware(driver storagedriver.StorageDriver, middlewares []co
|
|||
return driver, nil
|
||||
}
|
||||
|
||||
// uploadPurgeDefaultConfig provides a default configuration for upload
|
||||
// purging to be used in the absence of configuration in the
|
||||
// confifuration file
|
||||
func uploadPurgeDefaultConfig() map[interface{}]interface{} {
|
||||
config := map[interface{}]interface{}{}
|
||||
config["enabled"] = true
|
||||
config["age"] = "168h"
|
||||
config["interval"] = "24h"
|
||||
config["dryrun"] = false
|
||||
return config
|
||||
}
|
||||
|
||||
func badPurgeUploadConfig(reason string) {
|
||||
panic(fmt.Sprintf("Unable to parse upload purge configuration: %s", reason))
|
||||
}
|
||||
|
||||
// startUploadPurger schedules a goroutine which will periodically
|
||||
// check upload directories for old files and delete them
|
||||
func startUploadPurger(storageDriver storagedriver.StorageDriver, log ctxu.Logger) {
|
||||
rand.Seed(time.Now().Unix())
|
||||
jitter := time.Duration(rand.Int()%60) * time.Minute
|
||||
func startUploadPurger(storageDriver storagedriver.StorageDriver, log ctxu.Logger, config map[interface{}]interface{}) {
|
||||
if config["enabled"] == false {
|
||||
return
|
||||
}
|
||||
|
||||
// Start with reasonable defaults
|
||||
// TODO:(richardscothern) make configurable
|
||||
purgeAge := time.Duration(7 * 24 * time.Hour)
|
||||
timeBetweenPurges := time.Duration(1 * 24 * time.Hour)
|
||||
var purgeAgeDuration time.Duration
|
||||
var err error
|
||||
purgeAge, ok := config["age"]
|
||||
if ok {
|
||||
ageStr, ok := purgeAge.(string)
|
||||
if !ok {
|
||||
badPurgeUploadConfig("age is not a string")
|
||||
}
|
||||
purgeAgeDuration, err = time.ParseDuration(ageStr)
|
||||
if err != nil {
|
||||
badPurgeUploadConfig(fmt.Sprintf("Cannot parse duration: %s", err.Error()))
|
||||
}
|
||||
} else {
|
||||
badPurgeUploadConfig("age missing")
|
||||
}
|
||||
|
||||
var intervalDuration time.Duration
|
||||
interval, ok := config["interval"]
|
||||
if ok {
|
||||
intervalStr, ok := interval.(string)
|
||||
if !ok {
|
||||
badPurgeUploadConfig("interval is not a string")
|
||||
}
|
||||
|
||||
intervalDuration, err = time.ParseDuration(intervalStr)
|
||||
if err != nil {
|
||||
badPurgeUploadConfig(fmt.Sprintf("Cannot parse interval: %s", err.Error()))
|
||||
}
|
||||
} else {
|
||||
badPurgeUploadConfig("interval missing")
|
||||
}
|
||||
|
||||
var dryRunBool bool
|
||||
dryRun, ok := config["dryrun"]
|
||||
if ok {
|
||||
dryRunBool, ok = dryRun.(bool)
|
||||
if !ok {
|
||||
badPurgeUploadConfig("cannot parse dryrun")
|
||||
}
|
||||
} else {
|
||||
badPurgeUploadConfig("dryrun missing")
|
||||
}
|
||||
|
||||
go func() {
|
||||
rand.Seed(time.Now().Unix())
|
||||
jitter := time.Duration(rand.Int()%60) * time.Minute
|
||||
log.Infof("Starting upload purge in %s", jitter)
|
||||
time.Sleep(jitter)
|
||||
|
||||
for {
|
||||
storage.PurgeUploads(storageDriver, time.Now().Add(-purgeAge), true)
|
||||
log.Infof("Starting upload purge in %s", timeBetweenPurges)
|
||||
time.Sleep(timeBetweenPurges)
|
||||
storage.PurgeUploads(storageDriver, time.Now().Add(-purgeAgeDuration), !dryRunBool)
|
||||
log.Infof("Starting upload purge in %s", intervalDuration)
|
||||
time.Sleep(intervalDuration)
|
||||
}
|
||||
}()
|
||||
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue