containerd/cmd/dist/pull.go

232 lines
5.5 KiB
Go
Raw Normal View History

cmd/dist, cmd/ctr: end to end image pull With this changeset, we now have a proof of concept of end to end pull. Up to this point, the relationship between subsystems has been somewhat theoretical. We now leverage fetching, the snapshot drivers, the rootfs service, image metadata and the execution service, validating the proposed model for containerd. There are a few caveats, including the need to move some of the access into GRPC services, but the basic components are there. The first command we will cover here is `dist pull`. This is the analog of `docker pull` and `git pull`. It performs a full resource fetch for an image and unpacks the root filesystem into the snapshot drivers. An example follows: ``` console $ sudo ./bin/dist pull docker.io/library/redis:latest docker.io/library/redis:latest: resolved |++++++++++++++++++++++++++++++++++++++| manifest-sha256:4c8fb09e8d634ab823b1c125e64f0e1ceaf216025aa38283ea1b42997f1e8059: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:3b281f2bcae3b25c701d53a219924fffe79bdb74385340b73a539ed4020999c4: done |++++++++++++++++++++++++++++++++++++++| config-sha256:e4a35914679d05d25e2fccfd310fde1aa59ffbbf1b0b9d36f7b03db5ca0311b0: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:4b7726832aec75f0a742266c7190c4d2217492722dfd603406208eaa902648d8: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:338a7133395941c85087522582af182d2f6477dbf54ba769cb24ec4fd91d728f: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:83f12ff60ff1132d1e59845e26c41968406b4176c1a85a50506c954696b21570: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:693502eb7dfbc6b94964ae66ebc72d3e32facd981c72995b09794f1e87bac184: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:622732cddc347afc9360b4b04b46c6f758191a1dc73d007f95548658847ee67e: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:19a7e34366a6f558336c364693df538c38307484b729a36fede76432789f084f: done |++++++++++++++++++++++++++++++++++++++| elapsed: 1.6 s total: 0.0 B (0.0 B/s) INFO[0001] unpacking rootfs ``` Note that we haven't integrated rootfs unpacking into the status output, but we pretty much have what is in docker today (:P). We can see the result of our pull with the following: ```console $ sudo ./bin/dist images REF TYPE DIGEST SIZE docker.io/library/redis:latest application/vnd.docker.distribution.manifest.v2+json sha256:4c8fb09e8d634ab823b1c125e64f0e1ceaf216025aa38283ea1b42997f1e8059 1.8 kB ``` The above shows that we have an image called "docker.io/library/redis:latest" mapped to the given digest marked with a specific format. We get the size of the manifest right now, not the full image, but we can add more as we need it. For the most part, this is all that is needed, but a few tweaks to the model for naming may need to be added. Specifically, we may want to index under a few different names, including those qualified by hash or matched by tag versions. We can do more work in this area as we develop the metadata store. The name shown above can then be used to run the actual container image. We can do this with the following command: ```console $ sudo ./bin/ctr run --id foo docker.io/library/redis:latest /usr/local/bin/redis-server 1:C 17 Mar 17:20:25.316 # Warning: no config file specified, using the default config. In order to specify a config file use /usr/local/bin/redis-server /path/to/redis.conf 1:M 17 Mar 17:20:25.317 * Increased maximum number of open files to 10032 (it was originally set to 1024). _._ _.-``__ ''-._ _.-`` `. `_. ''-._ Redis 3.2.8 (00000000/0) 64 bit .-`` .-```. ```\/ _.,_ ''-._ ( ' , .-` | `, ) Running in standalone mode |`-._`-...-` __...-.``-._|'` _.-'| Port: 6379 | `-._ `._ / _.-' | PID: 1 `-._ `-._ `-./ _.-' _.-' |`-._`-._ `-.__.-' _.-'_.-'| | `-._`-._ _.-'_.-' | http://redis.io `-._ `-._`-.__.-'_.-' _.-' |`-._`-._ `-.__.-' _.-'_.-'| | `-._`-._ _.-'_.-' | `-._ `-._`-.__.-'_.-' _.-' `-._ `-.__.-' _.-' `-._ _.-' `-.__.-' 1:M 17 Mar 17:20:25.326 # WARNING: The TCP backlog setting of 511 cannot be enforced because /proc/sys/net/core/somaxconn is set to the lower value of 128. 1:M 17 Mar 17:20:25.326 # Server started, Redis version 3.2.8 1:M 17 Mar 17:20:25.326 # WARNING overcommit_memory is set to 0! Background save may fail under low memory condition. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect. 1:M 17 Mar 17:20:25.326 # WARNING you have Transparent Huge Pages (THP) support enabled in your kernel. This will create latency and memory usage issues with Redis. To fix this issue run the command 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' as root, and add it to your /etc/rc.local in order to retain the setting after a reboot. Redis must be restarted after THP is disabled. 1:M 17 Mar 17:20:25.326 * The server is now ready to accept connections on port 6379 ``` Wow! So, now we are running `redis`! There are still a few things to work out. Notice that we have to specify the command as part of the arguments to `ctr run`. This is because are not yet reading the image config and converting it to an OCI runtime config. With the base laid in this PR, adding such functionality should be straightforward. While this is a _little_ messy, this is great progress. It should be easy sailing from here. Signed-off-by: Stephen J Day <stephen.day@docker.com>
2017-03-18 00:00:52 +00:00
package main
import (
"context"
"encoding/json"
"os"
"text/tabwriter"
"time"
contentapi "github.com/containerd/containerd/api/services/content"
rootfsapi "github.com/containerd/containerd/api/services/rootfs"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/log"
"github.com/containerd/containerd/progress"
"github.com/containerd/containerd/remotes"
contentservice "github.com/containerd/containerd/services/content"
rootfsservice "github.com/containerd/containerd/services/rootfs"
cmd/dist, cmd/ctr: end to end image pull With this changeset, we now have a proof of concept of end to end pull. Up to this point, the relationship between subsystems has been somewhat theoretical. We now leverage fetching, the snapshot drivers, the rootfs service, image metadata and the execution service, validating the proposed model for containerd. There are a few caveats, including the need to move some of the access into GRPC services, but the basic components are there. The first command we will cover here is `dist pull`. This is the analog of `docker pull` and `git pull`. It performs a full resource fetch for an image and unpacks the root filesystem into the snapshot drivers. An example follows: ``` console $ sudo ./bin/dist pull docker.io/library/redis:latest docker.io/library/redis:latest: resolved |++++++++++++++++++++++++++++++++++++++| manifest-sha256:4c8fb09e8d634ab823b1c125e64f0e1ceaf216025aa38283ea1b42997f1e8059: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:3b281f2bcae3b25c701d53a219924fffe79bdb74385340b73a539ed4020999c4: done |++++++++++++++++++++++++++++++++++++++| config-sha256:e4a35914679d05d25e2fccfd310fde1aa59ffbbf1b0b9d36f7b03db5ca0311b0: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:4b7726832aec75f0a742266c7190c4d2217492722dfd603406208eaa902648d8: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:338a7133395941c85087522582af182d2f6477dbf54ba769cb24ec4fd91d728f: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:83f12ff60ff1132d1e59845e26c41968406b4176c1a85a50506c954696b21570: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:693502eb7dfbc6b94964ae66ebc72d3e32facd981c72995b09794f1e87bac184: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:622732cddc347afc9360b4b04b46c6f758191a1dc73d007f95548658847ee67e: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:19a7e34366a6f558336c364693df538c38307484b729a36fede76432789f084f: done |++++++++++++++++++++++++++++++++++++++| elapsed: 1.6 s total: 0.0 B (0.0 B/s) INFO[0001] unpacking rootfs ``` Note that we haven't integrated rootfs unpacking into the status output, but we pretty much have what is in docker today (:P). We can see the result of our pull with the following: ```console $ sudo ./bin/dist images REF TYPE DIGEST SIZE docker.io/library/redis:latest application/vnd.docker.distribution.manifest.v2+json sha256:4c8fb09e8d634ab823b1c125e64f0e1ceaf216025aa38283ea1b42997f1e8059 1.8 kB ``` The above shows that we have an image called "docker.io/library/redis:latest" mapped to the given digest marked with a specific format. We get the size of the manifest right now, not the full image, but we can add more as we need it. For the most part, this is all that is needed, but a few tweaks to the model for naming may need to be added. Specifically, we may want to index under a few different names, including those qualified by hash or matched by tag versions. We can do more work in this area as we develop the metadata store. The name shown above can then be used to run the actual container image. We can do this with the following command: ```console $ sudo ./bin/ctr run --id foo docker.io/library/redis:latest /usr/local/bin/redis-server 1:C 17 Mar 17:20:25.316 # Warning: no config file specified, using the default config. In order to specify a config file use /usr/local/bin/redis-server /path/to/redis.conf 1:M 17 Mar 17:20:25.317 * Increased maximum number of open files to 10032 (it was originally set to 1024). _._ _.-``__ ''-._ _.-`` `. `_. ''-._ Redis 3.2.8 (00000000/0) 64 bit .-`` .-```. ```\/ _.,_ ''-._ ( ' , .-` | `, ) Running in standalone mode |`-._`-...-` __...-.``-._|'` _.-'| Port: 6379 | `-._ `._ / _.-' | PID: 1 `-._ `-._ `-./ _.-' _.-' |`-._`-._ `-.__.-' _.-'_.-'| | `-._`-._ _.-'_.-' | http://redis.io `-._ `-._`-.__.-'_.-' _.-' |`-._`-._ `-.__.-' _.-'_.-'| | `-._`-._ _.-'_.-' | `-._ `-._`-.__.-'_.-' _.-' `-._ `-.__.-' _.-' `-._ _.-' `-.__.-' 1:M 17 Mar 17:20:25.326 # WARNING: The TCP backlog setting of 511 cannot be enforced because /proc/sys/net/core/somaxconn is set to the lower value of 128. 1:M 17 Mar 17:20:25.326 # Server started, Redis version 3.2.8 1:M 17 Mar 17:20:25.326 # WARNING overcommit_memory is set to 0! Background save may fail under low memory condition. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect. 1:M 17 Mar 17:20:25.326 # WARNING you have Transparent Huge Pages (THP) support enabled in your kernel. This will create latency and memory usage issues with Redis. To fix this issue run the command 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' as root, and add it to your /etc/rc.local in order to retain the setting after a reboot. Redis must be restarted after THP is disabled. 1:M 17 Mar 17:20:25.326 * The server is now ready to accept connections on port 6379 ``` Wow! So, now we are running `redis`! There are still a few things to work out. Notice that we have to specify the command as part of the arguments to `ctr run`. This is because are not yet reading the image config and converting it to an OCI runtime config. With the base laid in this PR, adding such functionality should be straightforward. While this is a _little_ messy, this is great progress. It should be easy sailing from here. Signed-off-by: Stephen J Day <stephen.day@docker.com>
2017-03-18 00:00:52 +00:00
"github.com/opencontainers/image-spec/identity"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/urfave/cli"
"golang.org/x/sync/errgroup"
)
var pullCommand = cli.Command{
Name: "pull",
Usage: "pull an image from a remote",
ArgsUsage: "[flags] <ref>",
Description: `Fetch and prepare an image for use in containerd.
After pulling an image, it should be ready to use the same reference in a run
command. As part of this process, we do the following:
1. Fetch all resources into containerd.
2. Prepare the snapshot filesystem with the pulled resources.
3. Register metadata for the image.
`,
Flags: []cli.Flag{},
Action: func(clicontext *cli.Context) error {
var (
ctx = background
ref = clicontext.Args().First()
)
conn, err := connectGRPC(clicontext)
if err != nil {
return err
}
imageStore, err := resolveImageStore(clicontext)
cmd/dist, cmd/ctr: end to end image pull With this changeset, we now have a proof of concept of end to end pull. Up to this point, the relationship between subsystems has been somewhat theoretical. We now leverage fetching, the snapshot drivers, the rootfs service, image metadata and the execution service, validating the proposed model for containerd. There are a few caveats, including the need to move some of the access into GRPC services, but the basic components are there. The first command we will cover here is `dist pull`. This is the analog of `docker pull` and `git pull`. It performs a full resource fetch for an image and unpacks the root filesystem into the snapshot drivers. An example follows: ``` console $ sudo ./bin/dist pull docker.io/library/redis:latest docker.io/library/redis:latest: resolved |++++++++++++++++++++++++++++++++++++++| manifest-sha256:4c8fb09e8d634ab823b1c125e64f0e1ceaf216025aa38283ea1b42997f1e8059: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:3b281f2bcae3b25c701d53a219924fffe79bdb74385340b73a539ed4020999c4: done |++++++++++++++++++++++++++++++++++++++| config-sha256:e4a35914679d05d25e2fccfd310fde1aa59ffbbf1b0b9d36f7b03db5ca0311b0: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:4b7726832aec75f0a742266c7190c4d2217492722dfd603406208eaa902648d8: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:338a7133395941c85087522582af182d2f6477dbf54ba769cb24ec4fd91d728f: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:83f12ff60ff1132d1e59845e26c41968406b4176c1a85a50506c954696b21570: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:693502eb7dfbc6b94964ae66ebc72d3e32facd981c72995b09794f1e87bac184: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:622732cddc347afc9360b4b04b46c6f758191a1dc73d007f95548658847ee67e: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:19a7e34366a6f558336c364693df538c38307484b729a36fede76432789f084f: done |++++++++++++++++++++++++++++++++++++++| elapsed: 1.6 s total: 0.0 B (0.0 B/s) INFO[0001] unpacking rootfs ``` Note that we haven't integrated rootfs unpacking into the status output, but we pretty much have what is in docker today (:P). We can see the result of our pull with the following: ```console $ sudo ./bin/dist images REF TYPE DIGEST SIZE docker.io/library/redis:latest application/vnd.docker.distribution.manifest.v2+json sha256:4c8fb09e8d634ab823b1c125e64f0e1ceaf216025aa38283ea1b42997f1e8059 1.8 kB ``` The above shows that we have an image called "docker.io/library/redis:latest" mapped to the given digest marked with a specific format. We get the size of the manifest right now, not the full image, but we can add more as we need it. For the most part, this is all that is needed, but a few tweaks to the model for naming may need to be added. Specifically, we may want to index under a few different names, including those qualified by hash or matched by tag versions. We can do more work in this area as we develop the metadata store. The name shown above can then be used to run the actual container image. We can do this with the following command: ```console $ sudo ./bin/ctr run --id foo docker.io/library/redis:latest /usr/local/bin/redis-server 1:C 17 Mar 17:20:25.316 # Warning: no config file specified, using the default config. In order to specify a config file use /usr/local/bin/redis-server /path/to/redis.conf 1:M 17 Mar 17:20:25.317 * Increased maximum number of open files to 10032 (it was originally set to 1024). _._ _.-``__ ''-._ _.-`` `. `_. ''-._ Redis 3.2.8 (00000000/0) 64 bit .-`` .-```. ```\/ _.,_ ''-._ ( ' , .-` | `, ) Running in standalone mode |`-._`-...-` __...-.``-._|'` _.-'| Port: 6379 | `-._ `._ / _.-' | PID: 1 `-._ `-._ `-./ _.-' _.-' |`-._`-._ `-.__.-' _.-'_.-'| | `-._`-._ _.-'_.-' | http://redis.io `-._ `-._`-.__.-'_.-' _.-' |`-._`-._ `-.__.-' _.-'_.-'| | `-._`-._ _.-'_.-' | `-._ `-._`-.__.-'_.-' _.-' `-._ `-.__.-' _.-' `-._ _.-' `-.__.-' 1:M 17 Mar 17:20:25.326 # WARNING: The TCP backlog setting of 511 cannot be enforced because /proc/sys/net/core/somaxconn is set to the lower value of 128. 1:M 17 Mar 17:20:25.326 # Server started, Redis version 3.2.8 1:M 17 Mar 17:20:25.326 # WARNING overcommit_memory is set to 0! Background save may fail under low memory condition. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect. 1:M 17 Mar 17:20:25.326 # WARNING you have Transparent Huge Pages (THP) support enabled in your kernel. This will create latency and memory usage issues with Redis. To fix this issue run the command 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' as root, and add it to your /etc/rc.local in order to retain the setting after a reboot. Redis must be restarted after THP is disabled. 1:M 17 Mar 17:20:25.326 * The server is now ready to accept connections on port 6379 ``` Wow! So, now we are running `redis`! There are still a few things to work out. Notice that we have to specify the command as part of the arguments to `ctr run`. This is because are not yet reading the image config and converting it to an OCI runtime config. With the base laid in this PR, adding such functionality should be straightforward. While this is a _little_ messy, this is great progress. It should be easy sailing from here. Signed-off-by: Stephen J Day <stephen.day@docker.com>
2017-03-18 00:00:52 +00:00
if err != nil {
return err
}
resolver, err := getResolver(ctx)
if err != nil {
return err
}
ongoing := newJobs()
// TODO(stevvooe): Must unify this type.
cmd/dist, cmd/ctr: end to end image pull With this changeset, we now have a proof of concept of end to end pull. Up to this point, the relationship between subsystems has been somewhat theoretical. We now leverage fetching, the snapshot drivers, the rootfs service, image metadata and the execution service, validating the proposed model for containerd. There are a few caveats, including the need to move some of the access into GRPC services, but the basic components are there. The first command we will cover here is `dist pull`. This is the analog of `docker pull` and `git pull`. It performs a full resource fetch for an image and unpacks the root filesystem into the snapshot drivers. An example follows: ``` console $ sudo ./bin/dist pull docker.io/library/redis:latest docker.io/library/redis:latest: resolved |++++++++++++++++++++++++++++++++++++++| manifest-sha256:4c8fb09e8d634ab823b1c125e64f0e1ceaf216025aa38283ea1b42997f1e8059: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:3b281f2bcae3b25c701d53a219924fffe79bdb74385340b73a539ed4020999c4: done |++++++++++++++++++++++++++++++++++++++| config-sha256:e4a35914679d05d25e2fccfd310fde1aa59ffbbf1b0b9d36f7b03db5ca0311b0: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:4b7726832aec75f0a742266c7190c4d2217492722dfd603406208eaa902648d8: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:338a7133395941c85087522582af182d2f6477dbf54ba769cb24ec4fd91d728f: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:83f12ff60ff1132d1e59845e26c41968406b4176c1a85a50506c954696b21570: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:693502eb7dfbc6b94964ae66ebc72d3e32facd981c72995b09794f1e87bac184: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:622732cddc347afc9360b4b04b46c6f758191a1dc73d007f95548658847ee67e: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:19a7e34366a6f558336c364693df538c38307484b729a36fede76432789f084f: done |++++++++++++++++++++++++++++++++++++++| elapsed: 1.6 s total: 0.0 B (0.0 B/s) INFO[0001] unpacking rootfs ``` Note that we haven't integrated rootfs unpacking into the status output, but we pretty much have what is in docker today (:P). We can see the result of our pull with the following: ```console $ sudo ./bin/dist images REF TYPE DIGEST SIZE docker.io/library/redis:latest application/vnd.docker.distribution.manifest.v2+json sha256:4c8fb09e8d634ab823b1c125e64f0e1ceaf216025aa38283ea1b42997f1e8059 1.8 kB ``` The above shows that we have an image called "docker.io/library/redis:latest" mapped to the given digest marked with a specific format. We get the size of the manifest right now, not the full image, but we can add more as we need it. For the most part, this is all that is needed, but a few tweaks to the model for naming may need to be added. Specifically, we may want to index under a few different names, including those qualified by hash or matched by tag versions. We can do more work in this area as we develop the metadata store. The name shown above can then be used to run the actual container image. We can do this with the following command: ```console $ sudo ./bin/ctr run --id foo docker.io/library/redis:latest /usr/local/bin/redis-server 1:C 17 Mar 17:20:25.316 # Warning: no config file specified, using the default config. In order to specify a config file use /usr/local/bin/redis-server /path/to/redis.conf 1:M 17 Mar 17:20:25.317 * Increased maximum number of open files to 10032 (it was originally set to 1024). _._ _.-``__ ''-._ _.-`` `. `_. ''-._ Redis 3.2.8 (00000000/0) 64 bit .-`` .-```. ```\/ _.,_ ''-._ ( ' , .-` | `, ) Running in standalone mode |`-._`-...-` __...-.``-._|'` _.-'| Port: 6379 | `-._ `._ / _.-' | PID: 1 `-._ `-._ `-./ _.-' _.-' |`-._`-._ `-.__.-' _.-'_.-'| | `-._`-._ _.-'_.-' | http://redis.io `-._ `-._`-.__.-'_.-' _.-' |`-._`-._ `-.__.-' _.-'_.-'| | `-._`-._ _.-'_.-' | `-._ `-._`-.__.-'_.-' _.-' `-._ `-.__.-' _.-' `-._ _.-' `-.__.-' 1:M 17 Mar 17:20:25.326 # WARNING: The TCP backlog setting of 511 cannot be enforced because /proc/sys/net/core/somaxconn is set to the lower value of 128. 1:M 17 Mar 17:20:25.326 # Server started, Redis version 3.2.8 1:M 17 Mar 17:20:25.326 # WARNING overcommit_memory is set to 0! Background save may fail under low memory condition. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect. 1:M 17 Mar 17:20:25.326 # WARNING you have Transparent Huge Pages (THP) support enabled in your kernel. This will create latency and memory usage issues with Redis. To fix this issue run the command 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' as root, and add it to your /etc/rc.local in order to retain the setting after a reboot. Redis must be restarted after THP is disabled. 1:M 17 Mar 17:20:25.326 * The server is now ready to accept connections on port 6379 ``` Wow! So, now we are running `redis`! There are still a few things to work out. Notice that we have to specify the command as part of the arguments to `ctr run`. This is because are not yet reading the image config and converting it to an OCI runtime config. With the base laid in this PR, adding such functionality should be straightforward. While this is a _little_ messy, this is great progress. It should be easy sailing from here. Signed-off-by: Stephen J Day <stephen.day@docker.com>
2017-03-18 00:00:52 +00:00
ingester := contentservice.NewIngesterFromClient(contentapi.NewContentClient(conn))
provider := contentservice.NewProviderFromClient(contentapi.NewContentClient(conn))
cmd/dist, cmd/ctr: end to end image pull With this changeset, we now have a proof of concept of end to end pull. Up to this point, the relationship between subsystems has been somewhat theoretical. We now leverage fetching, the snapshot drivers, the rootfs service, image metadata and the execution service, validating the proposed model for containerd. There are a few caveats, including the need to move some of the access into GRPC services, but the basic components are there. The first command we will cover here is `dist pull`. This is the analog of `docker pull` and `git pull`. It performs a full resource fetch for an image and unpacks the root filesystem into the snapshot drivers. An example follows: ``` console $ sudo ./bin/dist pull docker.io/library/redis:latest docker.io/library/redis:latest: resolved |++++++++++++++++++++++++++++++++++++++| manifest-sha256:4c8fb09e8d634ab823b1c125e64f0e1ceaf216025aa38283ea1b42997f1e8059: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:3b281f2bcae3b25c701d53a219924fffe79bdb74385340b73a539ed4020999c4: done |++++++++++++++++++++++++++++++++++++++| config-sha256:e4a35914679d05d25e2fccfd310fde1aa59ffbbf1b0b9d36f7b03db5ca0311b0: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:4b7726832aec75f0a742266c7190c4d2217492722dfd603406208eaa902648d8: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:338a7133395941c85087522582af182d2f6477dbf54ba769cb24ec4fd91d728f: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:83f12ff60ff1132d1e59845e26c41968406b4176c1a85a50506c954696b21570: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:693502eb7dfbc6b94964ae66ebc72d3e32facd981c72995b09794f1e87bac184: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:622732cddc347afc9360b4b04b46c6f758191a1dc73d007f95548658847ee67e: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:19a7e34366a6f558336c364693df538c38307484b729a36fede76432789f084f: done |++++++++++++++++++++++++++++++++++++++| elapsed: 1.6 s total: 0.0 B (0.0 B/s) INFO[0001] unpacking rootfs ``` Note that we haven't integrated rootfs unpacking into the status output, but we pretty much have what is in docker today (:P). We can see the result of our pull with the following: ```console $ sudo ./bin/dist images REF TYPE DIGEST SIZE docker.io/library/redis:latest application/vnd.docker.distribution.manifest.v2+json sha256:4c8fb09e8d634ab823b1c125e64f0e1ceaf216025aa38283ea1b42997f1e8059 1.8 kB ``` The above shows that we have an image called "docker.io/library/redis:latest" mapped to the given digest marked with a specific format. We get the size of the manifest right now, not the full image, but we can add more as we need it. For the most part, this is all that is needed, but a few tweaks to the model for naming may need to be added. Specifically, we may want to index under a few different names, including those qualified by hash or matched by tag versions. We can do more work in this area as we develop the metadata store. The name shown above can then be used to run the actual container image. We can do this with the following command: ```console $ sudo ./bin/ctr run --id foo docker.io/library/redis:latest /usr/local/bin/redis-server 1:C 17 Mar 17:20:25.316 # Warning: no config file specified, using the default config. In order to specify a config file use /usr/local/bin/redis-server /path/to/redis.conf 1:M 17 Mar 17:20:25.317 * Increased maximum number of open files to 10032 (it was originally set to 1024). _._ _.-``__ ''-._ _.-`` `. `_. ''-._ Redis 3.2.8 (00000000/0) 64 bit .-`` .-```. ```\/ _.,_ ''-._ ( ' , .-` | `, ) Running in standalone mode |`-._`-...-` __...-.``-._|'` _.-'| Port: 6379 | `-._ `._ / _.-' | PID: 1 `-._ `-._ `-./ _.-' _.-' |`-._`-._ `-.__.-' _.-'_.-'| | `-._`-._ _.-'_.-' | http://redis.io `-._ `-._`-.__.-'_.-' _.-' |`-._`-._ `-.__.-' _.-'_.-'| | `-._`-._ _.-'_.-' | `-._ `-._`-.__.-'_.-' _.-' `-._ `-.__.-' _.-' `-._ _.-' `-.__.-' 1:M 17 Mar 17:20:25.326 # WARNING: The TCP backlog setting of 511 cannot be enforced because /proc/sys/net/core/somaxconn is set to the lower value of 128. 1:M 17 Mar 17:20:25.326 # Server started, Redis version 3.2.8 1:M 17 Mar 17:20:25.326 # WARNING overcommit_memory is set to 0! Background save may fail under low memory condition. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect. 1:M 17 Mar 17:20:25.326 # WARNING you have Transparent Huge Pages (THP) support enabled in your kernel. This will create latency and memory usage issues with Redis. To fix this issue run the command 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' as root, and add it to your /etc/rc.local in order to retain the setting after a reboot. Redis must be restarted after THP is disabled. 1:M 17 Mar 17:20:25.326 * The server is now ready to accept connections on port 6379 ``` Wow! So, now we are running `redis`! There are still a few things to work out. Notice that we have to specify the command as part of the arguments to `ctr run`. This is because are not yet reading the image config and converting it to an OCI runtime config. With the base laid in this PR, adding such functionality should be straightforward. While this is a _little_ messy, this is great progress. It should be easy sailing from here. Signed-off-by: Stephen J Day <stephen.day@docker.com>
2017-03-18 00:00:52 +00:00
cs, err := resolveContentStore(clicontext)
if err != nil {
return err
}
eg, ctx := errgroup.WithContext(ctx)
var resolvedImageName string
resolved := make(chan struct{})
eg.Go(func() error {
ongoing.add(ref)
name, desc, fetcher, err := resolver.Resolve(ctx, ref)
if err != nil {
return err
}
log.G(ctx).WithField("image", name).Debug("fetching")
resolvedImageName = name
close(resolved)
eg.Go(func() error {
return imageStore.Put(ctx, name, desc)
cmd/dist, cmd/ctr: end to end image pull With this changeset, we now have a proof of concept of end to end pull. Up to this point, the relationship between subsystems has been somewhat theoretical. We now leverage fetching, the snapshot drivers, the rootfs service, image metadata and the execution service, validating the proposed model for containerd. There are a few caveats, including the need to move some of the access into GRPC services, but the basic components are there. The first command we will cover here is `dist pull`. This is the analog of `docker pull` and `git pull`. It performs a full resource fetch for an image and unpacks the root filesystem into the snapshot drivers. An example follows: ``` console $ sudo ./bin/dist pull docker.io/library/redis:latest docker.io/library/redis:latest: resolved |++++++++++++++++++++++++++++++++++++++| manifest-sha256:4c8fb09e8d634ab823b1c125e64f0e1ceaf216025aa38283ea1b42997f1e8059: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:3b281f2bcae3b25c701d53a219924fffe79bdb74385340b73a539ed4020999c4: done |++++++++++++++++++++++++++++++++++++++| config-sha256:e4a35914679d05d25e2fccfd310fde1aa59ffbbf1b0b9d36f7b03db5ca0311b0: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:4b7726832aec75f0a742266c7190c4d2217492722dfd603406208eaa902648d8: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:338a7133395941c85087522582af182d2f6477dbf54ba769cb24ec4fd91d728f: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:83f12ff60ff1132d1e59845e26c41968406b4176c1a85a50506c954696b21570: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:693502eb7dfbc6b94964ae66ebc72d3e32facd981c72995b09794f1e87bac184: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:622732cddc347afc9360b4b04b46c6f758191a1dc73d007f95548658847ee67e: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:19a7e34366a6f558336c364693df538c38307484b729a36fede76432789f084f: done |++++++++++++++++++++++++++++++++++++++| elapsed: 1.6 s total: 0.0 B (0.0 B/s) INFO[0001] unpacking rootfs ``` Note that we haven't integrated rootfs unpacking into the status output, but we pretty much have what is in docker today (:P). We can see the result of our pull with the following: ```console $ sudo ./bin/dist images REF TYPE DIGEST SIZE docker.io/library/redis:latest application/vnd.docker.distribution.manifest.v2+json sha256:4c8fb09e8d634ab823b1c125e64f0e1ceaf216025aa38283ea1b42997f1e8059 1.8 kB ``` The above shows that we have an image called "docker.io/library/redis:latest" mapped to the given digest marked with a specific format. We get the size of the manifest right now, not the full image, but we can add more as we need it. For the most part, this is all that is needed, but a few tweaks to the model for naming may need to be added. Specifically, we may want to index under a few different names, including those qualified by hash or matched by tag versions. We can do more work in this area as we develop the metadata store. The name shown above can then be used to run the actual container image. We can do this with the following command: ```console $ sudo ./bin/ctr run --id foo docker.io/library/redis:latest /usr/local/bin/redis-server 1:C 17 Mar 17:20:25.316 # Warning: no config file specified, using the default config. In order to specify a config file use /usr/local/bin/redis-server /path/to/redis.conf 1:M 17 Mar 17:20:25.317 * Increased maximum number of open files to 10032 (it was originally set to 1024). _._ _.-``__ ''-._ _.-`` `. `_. ''-._ Redis 3.2.8 (00000000/0) 64 bit .-`` .-```. ```\/ _.,_ ''-._ ( ' , .-` | `, ) Running in standalone mode |`-._`-...-` __...-.``-._|'` _.-'| Port: 6379 | `-._ `._ / _.-' | PID: 1 `-._ `-._ `-./ _.-' _.-' |`-._`-._ `-.__.-' _.-'_.-'| | `-._`-._ _.-'_.-' | http://redis.io `-._ `-._`-.__.-'_.-' _.-' |`-._`-._ `-.__.-' _.-'_.-'| | `-._`-._ _.-'_.-' | `-._ `-._`-.__.-'_.-' _.-' `-._ `-.__.-' _.-' `-._ _.-' `-.__.-' 1:M 17 Mar 17:20:25.326 # WARNING: The TCP backlog setting of 511 cannot be enforced because /proc/sys/net/core/somaxconn is set to the lower value of 128. 1:M 17 Mar 17:20:25.326 # Server started, Redis version 3.2.8 1:M 17 Mar 17:20:25.326 # WARNING overcommit_memory is set to 0! Background save may fail under low memory condition. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect. 1:M 17 Mar 17:20:25.326 # WARNING you have Transparent Huge Pages (THP) support enabled in your kernel. This will create latency and memory usage issues with Redis. To fix this issue run the command 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' as root, and add it to your /etc/rc.local in order to retain the setting after a reboot. Redis must be restarted after THP is disabled. 1:M 17 Mar 17:20:25.326 * The server is now ready to accept connections on port 6379 ``` Wow! So, now we are running `redis`! There are still a few things to work out. Notice that we have to specify the command as part of the arguments to `ctr run`. This is because are not yet reading the image config and converting it to an OCI runtime config. With the base laid in this PR, adding such functionality should be straightforward. While this is a _little_ messy, this is great progress. It should be easy sailing from here. Signed-off-by: Stephen J Day <stephen.day@docker.com>
2017-03-18 00:00:52 +00:00
})
return images.Dispatch(ctx,
images.Handlers(images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
cmd/dist, cmd/ctr: end to end image pull With this changeset, we now have a proof of concept of end to end pull. Up to this point, the relationship between subsystems has been somewhat theoretical. We now leverage fetching, the snapshot drivers, the rootfs service, image metadata and the execution service, validating the proposed model for containerd. There are a few caveats, including the need to move some of the access into GRPC services, but the basic components are there. The first command we will cover here is `dist pull`. This is the analog of `docker pull` and `git pull`. It performs a full resource fetch for an image and unpacks the root filesystem into the snapshot drivers. An example follows: ``` console $ sudo ./bin/dist pull docker.io/library/redis:latest docker.io/library/redis:latest: resolved |++++++++++++++++++++++++++++++++++++++| manifest-sha256:4c8fb09e8d634ab823b1c125e64f0e1ceaf216025aa38283ea1b42997f1e8059: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:3b281f2bcae3b25c701d53a219924fffe79bdb74385340b73a539ed4020999c4: done |++++++++++++++++++++++++++++++++++++++| config-sha256:e4a35914679d05d25e2fccfd310fde1aa59ffbbf1b0b9d36f7b03db5ca0311b0: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:4b7726832aec75f0a742266c7190c4d2217492722dfd603406208eaa902648d8: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:338a7133395941c85087522582af182d2f6477dbf54ba769cb24ec4fd91d728f: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:83f12ff60ff1132d1e59845e26c41968406b4176c1a85a50506c954696b21570: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:693502eb7dfbc6b94964ae66ebc72d3e32facd981c72995b09794f1e87bac184: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:622732cddc347afc9360b4b04b46c6f758191a1dc73d007f95548658847ee67e: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:19a7e34366a6f558336c364693df538c38307484b729a36fede76432789f084f: done |++++++++++++++++++++++++++++++++++++++| elapsed: 1.6 s total: 0.0 B (0.0 B/s) INFO[0001] unpacking rootfs ``` Note that we haven't integrated rootfs unpacking into the status output, but we pretty much have what is in docker today (:P). We can see the result of our pull with the following: ```console $ sudo ./bin/dist images REF TYPE DIGEST SIZE docker.io/library/redis:latest application/vnd.docker.distribution.manifest.v2+json sha256:4c8fb09e8d634ab823b1c125e64f0e1ceaf216025aa38283ea1b42997f1e8059 1.8 kB ``` The above shows that we have an image called "docker.io/library/redis:latest" mapped to the given digest marked with a specific format. We get the size of the manifest right now, not the full image, but we can add more as we need it. For the most part, this is all that is needed, but a few tweaks to the model for naming may need to be added. Specifically, we may want to index under a few different names, including those qualified by hash or matched by tag versions. We can do more work in this area as we develop the metadata store. The name shown above can then be used to run the actual container image. We can do this with the following command: ```console $ sudo ./bin/ctr run --id foo docker.io/library/redis:latest /usr/local/bin/redis-server 1:C 17 Mar 17:20:25.316 # Warning: no config file specified, using the default config. In order to specify a config file use /usr/local/bin/redis-server /path/to/redis.conf 1:M 17 Mar 17:20:25.317 * Increased maximum number of open files to 10032 (it was originally set to 1024). _._ _.-``__ ''-._ _.-`` `. `_. ''-._ Redis 3.2.8 (00000000/0) 64 bit .-`` .-```. ```\/ _.,_ ''-._ ( ' , .-` | `, ) Running in standalone mode |`-._`-...-` __...-.``-._|'` _.-'| Port: 6379 | `-._ `._ / _.-' | PID: 1 `-._ `-._ `-./ _.-' _.-' |`-._`-._ `-.__.-' _.-'_.-'| | `-._`-._ _.-'_.-' | http://redis.io `-._ `-._`-.__.-'_.-' _.-' |`-._`-._ `-.__.-' _.-'_.-'| | `-._`-._ _.-'_.-' | `-._ `-._`-.__.-'_.-' _.-' `-._ `-.__.-' _.-' `-._ _.-' `-.__.-' 1:M 17 Mar 17:20:25.326 # WARNING: The TCP backlog setting of 511 cannot be enforced because /proc/sys/net/core/somaxconn is set to the lower value of 128. 1:M 17 Mar 17:20:25.326 # Server started, Redis version 3.2.8 1:M 17 Mar 17:20:25.326 # WARNING overcommit_memory is set to 0! Background save may fail under low memory condition. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect. 1:M 17 Mar 17:20:25.326 # WARNING you have Transparent Huge Pages (THP) support enabled in your kernel. This will create latency and memory usage issues with Redis. To fix this issue run the command 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' as root, and add it to your /etc/rc.local in order to retain the setting after a reboot. Redis must be restarted after THP is disabled. 1:M 17 Mar 17:20:25.326 * The server is now ready to accept connections on port 6379 ``` Wow! So, now we are running `redis`! There are still a few things to work out. Notice that we have to specify the command as part of the arguments to `ctr run`. This is because are not yet reading the image config and converting it to an OCI runtime config. With the base laid in this PR, adding such functionality should be straightforward. While this is a _little_ messy, this is great progress. It should be easy sailing from here. Signed-off-by: Stephen J Day <stephen.day@docker.com>
2017-03-18 00:00:52 +00:00
ongoing.add(remotes.MakeRefKey(ctx, desc))
return nil, nil
}),
remotes.FetchHandler(ingester, fetcher),
images.ChildrenHandler(provider)),
cmd/dist, cmd/ctr: end to end image pull With this changeset, we now have a proof of concept of end to end pull. Up to this point, the relationship between subsystems has been somewhat theoretical. We now leverage fetching, the snapshot drivers, the rootfs service, image metadata and the execution service, validating the proposed model for containerd. There are a few caveats, including the need to move some of the access into GRPC services, but the basic components are there. The first command we will cover here is `dist pull`. This is the analog of `docker pull` and `git pull`. It performs a full resource fetch for an image and unpacks the root filesystem into the snapshot drivers. An example follows: ``` console $ sudo ./bin/dist pull docker.io/library/redis:latest docker.io/library/redis:latest: resolved |++++++++++++++++++++++++++++++++++++++| manifest-sha256:4c8fb09e8d634ab823b1c125e64f0e1ceaf216025aa38283ea1b42997f1e8059: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:3b281f2bcae3b25c701d53a219924fffe79bdb74385340b73a539ed4020999c4: done |++++++++++++++++++++++++++++++++++++++| config-sha256:e4a35914679d05d25e2fccfd310fde1aa59ffbbf1b0b9d36f7b03db5ca0311b0: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:4b7726832aec75f0a742266c7190c4d2217492722dfd603406208eaa902648d8: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:338a7133395941c85087522582af182d2f6477dbf54ba769cb24ec4fd91d728f: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:83f12ff60ff1132d1e59845e26c41968406b4176c1a85a50506c954696b21570: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:693502eb7dfbc6b94964ae66ebc72d3e32facd981c72995b09794f1e87bac184: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:622732cddc347afc9360b4b04b46c6f758191a1dc73d007f95548658847ee67e: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:19a7e34366a6f558336c364693df538c38307484b729a36fede76432789f084f: done |++++++++++++++++++++++++++++++++++++++| elapsed: 1.6 s total: 0.0 B (0.0 B/s) INFO[0001] unpacking rootfs ``` Note that we haven't integrated rootfs unpacking into the status output, but we pretty much have what is in docker today (:P). We can see the result of our pull with the following: ```console $ sudo ./bin/dist images REF TYPE DIGEST SIZE docker.io/library/redis:latest application/vnd.docker.distribution.manifest.v2+json sha256:4c8fb09e8d634ab823b1c125e64f0e1ceaf216025aa38283ea1b42997f1e8059 1.8 kB ``` The above shows that we have an image called "docker.io/library/redis:latest" mapped to the given digest marked with a specific format. We get the size of the manifest right now, not the full image, but we can add more as we need it. For the most part, this is all that is needed, but a few tweaks to the model for naming may need to be added. Specifically, we may want to index under a few different names, including those qualified by hash or matched by tag versions. We can do more work in this area as we develop the metadata store. The name shown above can then be used to run the actual container image. We can do this with the following command: ```console $ sudo ./bin/ctr run --id foo docker.io/library/redis:latest /usr/local/bin/redis-server 1:C 17 Mar 17:20:25.316 # Warning: no config file specified, using the default config. In order to specify a config file use /usr/local/bin/redis-server /path/to/redis.conf 1:M 17 Mar 17:20:25.317 * Increased maximum number of open files to 10032 (it was originally set to 1024). _._ _.-``__ ''-._ _.-`` `. `_. ''-._ Redis 3.2.8 (00000000/0) 64 bit .-`` .-```. ```\/ _.,_ ''-._ ( ' , .-` | `, ) Running in standalone mode |`-._`-...-` __...-.``-._|'` _.-'| Port: 6379 | `-._ `._ / _.-' | PID: 1 `-._ `-._ `-./ _.-' _.-' |`-._`-._ `-.__.-' _.-'_.-'| | `-._`-._ _.-'_.-' | http://redis.io `-._ `-._`-.__.-'_.-' _.-' |`-._`-._ `-.__.-' _.-'_.-'| | `-._`-._ _.-'_.-' | `-._ `-._`-.__.-'_.-' _.-' `-._ `-.__.-' _.-' `-._ _.-' `-.__.-' 1:M 17 Mar 17:20:25.326 # WARNING: The TCP backlog setting of 511 cannot be enforced because /proc/sys/net/core/somaxconn is set to the lower value of 128. 1:M 17 Mar 17:20:25.326 # Server started, Redis version 3.2.8 1:M 17 Mar 17:20:25.326 # WARNING overcommit_memory is set to 0! Background save may fail under low memory condition. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect. 1:M 17 Mar 17:20:25.326 # WARNING you have Transparent Huge Pages (THP) support enabled in your kernel. This will create latency and memory usage issues with Redis. To fix this issue run the command 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' as root, and add it to your /etc/rc.local in order to retain the setting after a reboot. Redis must be restarted after THP is disabled. 1:M 17 Mar 17:20:25.326 * The server is now ready to accept connections on port 6379 ``` Wow! So, now we are running `redis`! There are still a few things to work out. Notice that we have to specify the command as part of the arguments to `ctr run`. This is because are not yet reading the image config and converting it to an OCI runtime config. With the base laid in this PR, adding such functionality should be straightforward. While this is a _little_ messy, this is great progress. It should be easy sailing from here. Signed-off-by: Stephen J Day <stephen.day@docker.com>
2017-03-18 00:00:52 +00:00
desc)
})
errs := make(chan error)
go func() {
defer close(errs)
errs <- eg.Wait()
}()
defer func() {
ctx := background
cmd/dist, cmd/ctr: end to end image pull With this changeset, we now have a proof of concept of end to end pull. Up to this point, the relationship between subsystems has been somewhat theoretical. We now leverage fetching, the snapshot drivers, the rootfs service, image metadata and the execution service, validating the proposed model for containerd. There are a few caveats, including the need to move some of the access into GRPC services, but the basic components are there. The first command we will cover here is `dist pull`. This is the analog of `docker pull` and `git pull`. It performs a full resource fetch for an image and unpacks the root filesystem into the snapshot drivers. An example follows: ``` console $ sudo ./bin/dist pull docker.io/library/redis:latest docker.io/library/redis:latest: resolved |++++++++++++++++++++++++++++++++++++++| manifest-sha256:4c8fb09e8d634ab823b1c125e64f0e1ceaf216025aa38283ea1b42997f1e8059: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:3b281f2bcae3b25c701d53a219924fffe79bdb74385340b73a539ed4020999c4: done |++++++++++++++++++++++++++++++++++++++| config-sha256:e4a35914679d05d25e2fccfd310fde1aa59ffbbf1b0b9d36f7b03db5ca0311b0: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:4b7726832aec75f0a742266c7190c4d2217492722dfd603406208eaa902648d8: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:338a7133395941c85087522582af182d2f6477dbf54ba769cb24ec4fd91d728f: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:83f12ff60ff1132d1e59845e26c41968406b4176c1a85a50506c954696b21570: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:693502eb7dfbc6b94964ae66ebc72d3e32facd981c72995b09794f1e87bac184: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:622732cddc347afc9360b4b04b46c6f758191a1dc73d007f95548658847ee67e: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:19a7e34366a6f558336c364693df538c38307484b729a36fede76432789f084f: done |++++++++++++++++++++++++++++++++++++++| elapsed: 1.6 s total: 0.0 B (0.0 B/s) INFO[0001] unpacking rootfs ``` Note that we haven't integrated rootfs unpacking into the status output, but we pretty much have what is in docker today (:P). We can see the result of our pull with the following: ```console $ sudo ./bin/dist images REF TYPE DIGEST SIZE docker.io/library/redis:latest application/vnd.docker.distribution.manifest.v2+json sha256:4c8fb09e8d634ab823b1c125e64f0e1ceaf216025aa38283ea1b42997f1e8059 1.8 kB ``` The above shows that we have an image called "docker.io/library/redis:latest" mapped to the given digest marked with a specific format. We get the size of the manifest right now, not the full image, but we can add more as we need it. For the most part, this is all that is needed, but a few tweaks to the model for naming may need to be added. Specifically, we may want to index under a few different names, including those qualified by hash or matched by tag versions. We can do more work in this area as we develop the metadata store. The name shown above can then be used to run the actual container image. We can do this with the following command: ```console $ sudo ./bin/ctr run --id foo docker.io/library/redis:latest /usr/local/bin/redis-server 1:C 17 Mar 17:20:25.316 # Warning: no config file specified, using the default config. In order to specify a config file use /usr/local/bin/redis-server /path/to/redis.conf 1:M 17 Mar 17:20:25.317 * Increased maximum number of open files to 10032 (it was originally set to 1024). _._ _.-``__ ''-._ _.-`` `. `_. ''-._ Redis 3.2.8 (00000000/0) 64 bit .-`` .-```. ```\/ _.,_ ''-._ ( ' , .-` | `, ) Running in standalone mode |`-._`-...-` __...-.``-._|'` _.-'| Port: 6379 | `-._ `._ / _.-' | PID: 1 `-._ `-._ `-./ _.-' _.-' |`-._`-._ `-.__.-' _.-'_.-'| | `-._`-._ _.-'_.-' | http://redis.io `-._ `-._`-.__.-'_.-' _.-' |`-._`-._ `-.__.-' _.-'_.-'| | `-._`-._ _.-'_.-' | `-._ `-._`-.__.-'_.-' _.-' `-._ `-.__.-' _.-' `-._ _.-' `-.__.-' 1:M 17 Mar 17:20:25.326 # WARNING: The TCP backlog setting of 511 cannot be enforced because /proc/sys/net/core/somaxconn is set to the lower value of 128. 1:M 17 Mar 17:20:25.326 # Server started, Redis version 3.2.8 1:M 17 Mar 17:20:25.326 # WARNING overcommit_memory is set to 0! Background save may fail under low memory condition. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect. 1:M 17 Mar 17:20:25.326 # WARNING you have Transparent Huge Pages (THP) support enabled in your kernel. This will create latency and memory usage issues with Redis. To fix this issue run the command 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' as root, and add it to your /etc/rc.local in order to retain the setting after a reboot. Redis must be restarted after THP is disabled. 1:M 17 Mar 17:20:25.326 * The server is now ready to accept connections on port 6379 ``` Wow! So, now we are running `redis`! There are still a few things to work out. Notice that we have to specify the command as part of the arguments to `ctr run`. This is because are not yet reading the image config and converting it to an OCI runtime config. With the base laid in this PR, adding such functionality should be straightforward. While this is a _little_ messy, this is great progress. It should be easy sailing from here. Signed-off-by: Stephen J Day <stephen.day@docker.com>
2017-03-18 00:00:52 +00:00
// TODO(stevvooe): This section unpacks the layers and resolves the
// root filesystem chainid for the image. For now, we just print
// it, but we should keep track of this in the metadata storage.
image, err := imageStore.Get(ctx, resolvedImageName)
cmd/dist, cmd/ctr: end to end image pull With this changeset, we now have a proof of concept of end to end pull. Up to this point, the relationship between subsystems has been somewhat theoretical. We now leverage fetching, the snapshot drivers, the rootfs service, image metadata and the execution service, validating the proposed model for containerd. There are a few caveats, including the need to move some of the access into GRPC services, but the basic components are there. The first command we will cover here is `dist pull`. This is the analog of `docker pull` and `git pull`. It performs a full resource fetch for an image and unpacks the root filesystem into the snapshot drivers. An example follows: ``` console $ sudo ./bin/dist pull docker.io/library/redis:latest docker.io/library/redis:latest: resolved |++++++++++++++++++++++++++++++++++++++| manifest-sha256:4c8fb09e8d634ab823b1c125e64f0e1ceaf216025aa38283ea1b42997f1e8059: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:3b281f2bcae3b25c701d53a219924fffe79bdb74385340b73a539ed4020999c4: done |++++++++++++++++++++++++++++++++++++++| config-sha256:e4a35914679d05d25e2fccfd310fde1aa59ffbbf1b0b9d36f7b03db5ca0311b0: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:4b7726832aec75f0a742266c7190c4d2217492722dfd603406208eaa902648d8: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:338a7133395941c85087522582af182d2f6477dbf54ba769cb24ec4fd91d728f: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:83f12ff60ff1132d1e59845e26c41968406b4176c1a85a50506c954696b21570: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:693502eb7dfbc6b94964ae66ebc72d3e32facd981c72995b09794f1e87bac184: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:622732cddc347afc9360b4b04b46c6f758191a1dc73d007f95548658847ee67e: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:19a7e34366a6f558336c364693df538c38307484b729a36fede76432789f084f: done |++++++++++++++++++++++++++++++++++++++| elapsed: 1.6 s total: 0.0 B (0.0 B/s) INFO[0001] unpacking rootfs ``` Note that we haven't integrated rootfs unpacking into the status output, but we pretty much have what is in docker today (:P). We can see the result of our pull with the following: ```console $ sudo ./bin/dist images REF TYPE DIGEST SIZE docker.io/library/redis:latest application/vnd.docker.distribution.manifest.v2+json sha256:4c8fb09e8d634ab823b1c125e64f0e1ceaf216025aa38283ea1b42997f1e8059 1.8 kB ``` The above shows that we have an image called "docker.io/library/redis:latest" mapped to the given digest marked with a specific format. We get the size of the manifest right now, not the full image, but we can add more as we need it. For the most part, this is all that is needed, but a few tweaks to the model for naming may need to be added. Specifically, we may want to index under a few different names, including those qualified by hash or matched by tag versions. We can do more work in this area as we develop the metadata store. The name shown above can then be used to run the actual container image. We can do this with the following command: ```console $ sudo ./bin/ctr run --id foo docker.io/library/redis:latest /usr/local/bin/redis-server 1:C 17 Mar 17:20:25.316 # Warning: no config file specified, using the default config. In order to specify a config file use /usr/local/bin/redis-server /path/to/redis.conf 1:M 17 Mar 17:20:25.317 * Increased maximum number of open files to 10032 (it was originally set to 1024). _._ _.-``__ ''-._ _.-`` `. `_. ''-._ Redis 3.2.8 (00000000/0) 64 bit .-`` .-```. ```\/ _.,_ ''-._ ( ' , .-` | `, ) Running in standalone mode |`-._`-...-` __...-.``-._|'` _.-'| Port: 6379 | `-._ `._ / _.-' | PID: 1 `-._ `-._ `-./ _.-' _.-' |`-._`-._ `-.__.-' _.-'_.-'| | `-._`-._ _.-'_.-' | http://redis.io `-._ `-._`-.__.-'_.-' _.-' |`-._`-._ `-.__.-' _.-'_.-'| | `-._`-._ _.-'_.-' | `-._ `-._`-.__.-'_.-' _.-' `-._ `-.__.-' _.-' `-._ _.-' `-.__.-' 1:M 17 Mar 17:20:25.326 # WARNING: The TCP backlog setting of 511 cannot be enforced because /proc/sys/net/core/somaxconn is set to the lower value of 128. 1:M 17 Mar 17:20:25.326 # Server started, Redis version 3.2.8 1:M 17 Mar 17:20:25.326 # WARNING overcommit_memory is set to 0! Background save may fail under low memory condition. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect. 1:M 17 Mar 17:20:25.326 # WARNING you have Transparent Huge Pages (THP) support enabled in your kernel. This will create latency and memory usage issues with Redis. To fix this issue run the command 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' as root, and add it to your /etc/rc.local in order to retain the setting after a reboot. Redis must be restarted after THP is disabled. 1:M 17 Mar 17:20:25.326 * The server is now ready to accept connections on port 6379 ``` Wow! So, now we are running `redis`! There are still a few things to work out. Notice that we have to specify the command as part of the arguments to `ctr run`. This is because are not yet reading the image config and converting it to an OCI runtime config. With the base laid in this PR, adding such functionality should be straightforward. While this is a _little_ messy, this is great progress. It should be easy sailing from here. Signed-off-by: Stephen J Day <stephen.day@docker.com>
2017-03-18 00:00:52 +00:00
if err != nil {
log.G(ctx).Fatal(err)
}
provider := contentservice.NewProviderFromClient(contentapi.NewContentClient(conn))
p, err := content.ReadBlob(ctx, provider, image.Target.Digest)
cmd/dist, cmd/ctr: end to end image pull With this changeset, we now have a proof of concept of end to end pull. Up to this point, the relationship between subsystems has been somewhat theoretical. We now leverage fetching, the snapshot drivers, the rootfs service, image metadata and the execution service, validating the proposed model for containerd. There are a few caveats, including the need to move some of the access into GRPC services, but the basic components are there. The first command we will cover here is `dist pull`. This is the analog of `docker pull` and `git pull`. It performs a full resource fetch for an image and unpacks the root filesystem into the snapshot drivers. An example follows: ``` console $ sudo ./bin/dist pull docker.io/library/redis:latest docker.io/library/redis:latest: resolved |++++++++++++++++++++++++++++++++++++++| manifest-sha256:4c8fb09e8d634ab823b1c125e64f0e1ceaf216025aa38283ea1b42997f1e8059: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:3b281f2bcae3b25c701d53a219924fffe79bdb74385340b73a539ed4020999c4: done |++++++++++++++++++++++++++++++++++++++| config-sha256:e4a35914679d05d25e2fccfd310fde1aa59ffbbf1b0b9d36f7b03db5ca0311b0: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:4b7726832aec75f0a742266c7190c4d2217492722dfd603406208eaa902648d8: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:338a7133395941c85087522582af182d2f6477dbf54ba769cb24ec4fd91d728f: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:83f12ff60ff1132d1e59845e26c41968406b4176c1a85a50506c954696b21570: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:693502eb7dfbc6b94964ae66ebc72d3e32facd981c72995b09794f1e87bac184: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:622732cddc347afc9360b4b04b46c6f758191a1dc73d007f95548658847ee67e: done |++++++++++++++++++++++++++++++++++++++| layer-sha256:19a7e34366a6f558336c364693df538c38307484b729a36fede76432789f084f: done |++++++++++++++++++++++++++++++++++++++| elapsed: 1.6 s total: 0.0 B (0.0 B/s) INFO[0001] unpacking rootfs ``` Note that we haven't integrated rootfs unpacking into the status output, but we pretty much have what is in docker today (:P). We can see the result of our pull with the following: ```console $ sudo ./bin/dist images REF TYPE DIGEST SIZE docker.io/library/redis:latest application/vnd.docker.distribution.manifest.v2+json sha256:4c8fb09e8d634ab823b1c125e64f0e1ceaf216025aa38283ea1b42997f1e8059 1.8 kB ``` The above shows that we have an image called "docker.io/library/redis:latest" mapped to the given digest marked with a specific format. We get the size of the manifest right now, not the full image, but we can add more as we need it. For the most part, this is all that is needed, but a few tweaks to the model for naming may need to be added. Specifically, we may want to index under a few different names, including those qualified by hash or matched by tag versions. We can do more work in this area as we develop the metadata store. The name shown above can then be used to run the actual container image. We can do this with the following command: ```console $ sudo ./bin/ctr run --id foo docker.io/library/redis:latest /usr/local/bin/redis-server 1:C 17 Mar 17:20:25.316 # Warning: no config file specified, using the default config. In order to specify a config file use /usr/local/bin/redis-server /path/to/redis.conf 1:M 17 Mar 17:20:25.317 * Increased maximum number of open files to 10032 (it was originally set to 1024). _._ _.-``__ ''-._ _.-`` `. `_. ''-._ Redis 3.2.8 (00000000/0) 64 bit .-`` .-```. ```\/ _.,_ ''-._ ( ' , .-` | `, ) Running in standalone mode |`-._`-...-` __...-.``-._|'` _.-'| Port: 6379 | `-._ `._ / _.-' | PID: 1 `-._ `-._ `-./ _.-' _.-' |`-._`-._ `-.__.-' _.-'_.-'| | `-._`-._ _.-'_.-' | http://redis.io `-._ `-._`-.__.-'_.-' _.-' |`-._`-._ `-.__.-' _.-'_.-'| | `-._`-._ _.-'_.-' | `-._ `-._`-.__.-'_.-' _.-' `-._ `-.__.-' _.-' `-._ _.-' `-.__.-' 1:M 17 Mar 17:20:25.326 # WARNING: The TCP backlog setting of 511 cannot be enforced because /proc/sys/net/core/somaxconn is set to the lower value of 128. 1:M 17 Mar 17:20:25.326 # Server started, Redis version 3.2.8 1:M 17 Mar 17:20:25.326 # WARNING overcommit_memory is set to 0! Background save may fail under low memory condition. To fix this issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this to take effect. 1:M 17 Mar 17:20:25.326 # WARNING you have Transparent Huge Pages (THP) support enabled in your kernel. This will create latency and memory usage issues with Redis. To fix this issue run the command 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' as root, and add it to your /etc/rc.local in order to retain the setting after a reboot. Redis must be restarted after THP is disabled. 1:M 17 Mar 17:20:25.326 * The server is now ready to accept connections on port 6379 ``` Wow! So, now we are running `redis`! There are still a few things to work out. Notice that we have to specify the command as part of the arguments to `ctr run`. This is because are not yet reading the image config and converting it to an OCI runtime config. With the base laid in this PR, adding such functionality should be straightforward. While this is a _little_ messy, this is great progress. It should be easy sailing from here. Signed-off-by: Stephen J Day <stephen.day@docker.com>
2017-03-18 00:00:52 +00:00
if err != nil {
log.G(ctx).Fatal(err)
}
var manifest ocispec.Manifest
if err := json.Unmarshal(p, &manifest); err != nil {
log.G(ctx).Fatal(err)
}
rootfs := rootfsservice.NewUnpackerFromClient(rootfsapi.NewRootFSClient(conn))
log.G(ctx).Info("unpacking rootfs")
chainID, err := rootfs.Unpack(ctx, manifest.Layers)
if err != nil {
log.G(ctx).Fatal(err)
}
diffIDs, err := image.RootFS(ctx, provider)
if err != nil {
log.G(ctx).WithError(err).Fatal("failed resolving rootfs")
}
expectedChainID := identity.ChainID(diffIDs)
if expectedChainID != chainID {
log.G(ctx).Fatal("rootfs service did not match chainid")
}
}()
var (
ticker = time.NewTicker(100 * time.Millisecond)
fw = progress.NewWriter(os.Stdout)
start = time.Now()
done bool
)
defer ticker.Stop()
for {
select {
case <-ticker.C:
fw.Flush()
tw := tabwriter.NewWriter(fw, 1, 8, 1, ' ', 0)
js := ongoing.jobs()
statuses := map[string]statusInfo{}
activeSeen := map[string]struct{}{}
if !done {
active, err := cs.Active()
if err != nil {
log.G(ctx).WithError(err).Error("active check failed")
continue
}
// update status of active entries!
for _, active := range active {
statuses[active.Ref] = statusInfo{
Ref: active.Ref,
Status: "downloading",
Offset: active.Offset,
Total: active.Total,
StartedAt: active.StartedAt,
UpdatedAt: active.UpdatedAt,
}
activeSeen[active.Ref] = struct{}{}
}
}
// now, update the items in jobs that are not in active
for _, j := range js {
if _, ok := activeSeen[j]; ok {
continue
}
status := "done"
if j == ref {
select {
case <-resolved:
status = "resolved"
default:
status = "resolving"
}
}
statuses[j] = statusInfo{
Ref: j,
Status: status, // for now!
}
}
var ordered []statusInfo
for _, j := range js {
ordered = append(ordered, statuses[j])
}
display(tw, ordered, start)
tw.Flush()
if done {
fw.Flush()
return nil
}
case err := <-errs:
if err != nil {
return err
}
done = true
case <-ctx.Done():
done = true // allow ui to update once more
}
}
},
}