Passive FIlesystem VAlidation

bit.ly/asg2018-vbatts-pfv

Vincent Batts  @vbatts

$> finger $(whoami)
Login: vbatts                           Name: Vincent Batts
Directory: /home/vbatts                 Shell: /bin/bash
Such mail.
Plan:
OHMAN
$> id -Gn
devel opencontainers docker appc redhat golang slackware
  • Packaging
  • Content Addressibility
  • Compression!
  • Reproducible Archives
  • Verify at rest filesystems

Agenda

Packaging

tar archives

Slackware packages (tar(1) archives)

Debian *.deb (ar(1) archive of tar(1) archives)

Red Hat *.rpm (custom key/value binary and cpio(1))

Java *.jar and *.war (zip(1) archive)

Ruby *.gem (tar(1) archive of tar(1) archives)

Container Images (tar(1) archives)

Content Addressibility

Opaque Object storage

changed object = new object

cryptographic assurance

compression!

inflate/deflate (RFC1951)

same objects, but variation in compression

Gzip (RFC1952)

`gzip` vs Golang `compress/gzip` vs Zlib

ideally compress for transfer and storage, but not for identity

compression!

#!/bin/sh
dd if=/dev/urandom of=rando.img bs=1M count=2
cat rando.img | gzip -n > rando.img.gz
cat rando.img | gzip -n -9 > rando.img.9.gz
cat rando.img | xz > rando.img.xz
cat rando.img | xz -9 > rando.img.9.xz
sha1sum rando.img* > SHA1

cat rando.img | gzip -n > rando.img.gz
cat rando.img | gzip -n -9 > rando.img.9.gz
cat rando.img | xz > rando.img.xz
cat rando.img | xz -9 > rando.img.9.xz
sha1sum -c ./SHA1

compression!

#!/usr/bin/env ruby

require 'zlib'
include Zlib

input = File.open(ARGV.first)
GzipWriter.open(ARGV.first + '.gz', DEFAULT_COMPRESSION, HUFFMAN_ONLY) do |gz|
  gz.write(IO.binread(input))
end
input.flush()
input.close()

compression!

package main
  
import (
        "compress/gzip"  
        "io"
        "os"  
)

func main() {
        input, err := os.Open(os.Args[1])
        if err != nil {
                println(err.Error())
                os.Exit(1)
        }
        output, err := os.Create(os.Args[1] + ".gz")
        if err != nil {
                println(err.Error())
                os.Exit(1)
        }
        gz := gzip.NewWriter(output)
        if _, err := io.Copy(gz, input); err != nil {
                println(err.Error())
                os.Exit(1)
        }
}

Verify at rest Filesystems

Regardless of transport, ensure resulting filesystem

(*.tar archive, rsync, bittorrent, IPFS, etc)

`rpm -qV <package>` functionality

Future hopes could be IMA/EVM

Passive validation of directory hierarchies

BSD mtree(8)

dm-verity in select use-cases

Verify at rest Filesystems

FreeBSD mtree(8)

mtree-port (for linux)

go-mtree (golang cli and library)

libarchive-formats(5) (so `bsdtar`)

casync mtree

umoci unpack

Verify at rest Filesystems

[root@host /]# gomtree -c -K sha256 -p /usr/ | head -30
#          user: root
#       machine: host
#          tree: /usr
#          date: Wed Sep 26 16:07:53 2018
#      keywords: size,type,uid,gid,mode,link,nlink,time,sha256digest

# .
/set type=file nlink=1 mode=0664 uid=0 gid=0
. size=100 type=dir mode=0755 time=1524747817.000000000
    tmp size=10 type=link mode=0777 link=../var/tmp time=1517996467.000000000

# bin
bin size=5268 type=dir mode=0555 time=1537977678.074646319
    \133 size=48 mode=0555 time=1521637266.000000000 sha256digest=afd97bbd643bfe1473794af167cd5c6f44fe449681033e3584b40b836f624b4b
    alias size=29 mode=0755 time=1521122304.000000000 sha256digest=c9e358c5012c2cf9171ec4f7692ac3a1cbc280617b610d77d813653a1d0dfeb8
    applydeltarpm size=72752 mode=0755 time=1517985721.000000000 sha256digest=359f076a0a259bdab3d8139546319efe076fe49733c44aae1a823872e51451eb
    arch size=51 mode=0555 time=1521637267.000000000 sha256digest=209bae4071910ef54b4a3bd302059bf7e00870d8bacffcd7c5489425f37ed16f

gomtree

Verify at rest Filesystems

[root@host /]# casync mtree /usr/
. type=dir mode=0755 uid=0 gid=0 time=1524747817.000000000
bin type=dir mode=0555 uid=0 gid=0 time=1537977212.955190222
bin/[ type=file mode=0555 size=48 uid=0 gid=0 time=1521637266.000000000 sha512256digest=f02fe283c47d25e8273a4ce6297eef3e9f2300f597207fa6b82fc12694bd9771
bin/alias type=file mode=0755 size=29 uid=0 gid=0 time=1521122304.000000000 sha512256digest=c526d0d764732aa40fa866033bc49882f1e575ff89f969261eb354945db57d75
bin/applydeltarpm type=file mode=0755 size=72752 uid=0 gid=0 time=1517985721.000000000 sha512256digest=794cd98d3aac69cd3a2ab0e6820e0070555fdbc79de906a79addb98b919baa47
bin/arch type=file mode=0555 size=51 uid=0 gid=0 time=1521637267.000000000 sha512256digest=5516a6b4ee54e9f5fa7d8842781bcb6b3799e2dfa581ce1941c1c9f06a22d648
bin/awk type=link mode=0777 link=gawk uid=0 gid=0 time=1519649348.000000000
bin/b2sum type=file mode=0555 size=52 uid=0 gid=0 time=1521637266.000000000 sha512256digest=423c318bb3cad9826bf3b932baccb66baf2c7f0d55cb0d7d10f41d7e4ebb8fe6
bin/base32 type=file mode=0555 size=53 uid=0 gid=0 time=1521637266.000000000 sha512256digest=45b43efd608aca559aa0dea075a46a554f7e7fcc3534ea2c7f8edc344a342ead
bin/base64 type=file mode=0555 size=53 uid=0 gid=0 time=1521637266.000000000 sha512256digest=d63d00e263f95f09a27ddd4fc6a00b76835e5371af0a2c9a32e3929305810536
[...]

casync mtree

Verify at rest Filesystems

> skopeo copy docker://docker.io/busybox:latest oci:busybox:latest
> umoci unpack --image ./busybox:latest busybox-bundle
> cat busybox-bundle/sha256_9b9b48e2d92691f344ad1701e7df04f89dd2041f2f05cccfec39af8ac3a62d25.mtree
#          user: root
#       machine: host
#          tree: busybox-bundle/rootfs
#          date: Wed Sep 26 16:21:46 2018
#      keywords: size,type,uid,gid,mode,link,nlink,tar_time,sha256digest,xattr

# .
/set type=file nlink=1 mode=0664 uid=0 gid=0
. size=52 type=dir mode=0755 tar_time=0.000000000

# bin
bin size=4638 type=dir mode=0755 tar_time=1533068407.000000000
    \133 size=1083720 mode=0755 nlink=393 tar_time=1533068407.000000000 sha256digest=ee8001feee768df41ff9a2b704b4a027d7ed76f78f09648a5b7ad9f1d7d45640
    \133\133 size=1083720 mode=0755 nlink=393 tar_time=1533068407.000000000 sha256digest=ee8001feee768df41ff9a2b704b4a027d7ed76f78f09648a5b7ad9f1d7d45640
    acpid size=1083720 mode=0755 nlink=393 tar_time=1533068407.000000000 sha256digest=ee8001feee768df41ff9a2b704b4a027d7ed76f78f09648a5b7ad9f1d7d45640
    add-shell size=1083720 mode=0755 nlink=393 tar_time=1533068407.000000000 sha256digest=ee8001feee768df41ff9a2b704b4a027d7ed76f78f09648a5b7ad9f1d7d45640
    addgroup size=1083720 mode=0755 nlink=393 tar_time=1533068407.000000000 sha256digest=ee8001feee768df41ff9a2b704b4a027d7ed76f78f09648a5b7ad9f1d7d45640
    adduser size=1083720 mode=0755 nlink=393 tar_time=1533068407.000000000 sha256digest=ee8001feee768df41ff9a2b704b4a027d7ed76f78f09648a5b7ad9f1d7d45640
    adjtimex size=1083720 mode=0755 nlink=393 tar_time=1533068407.000000000 sha256digest=ee8001feee768df41ff9a2b704b4a027d7ed76f78f09648a5b7ad9f1d7d45640
[...]

umoci unpack

Verify at rest Filesystems

tar cf /tmp/demo.tar .
gomtree -c -T /tmp/demo.tar -K sha256digest | tee /tmp/demo.mtree

gomtree -f /tmp/demo.mtree -T /tmp/demo.tar
echo $?

read

gomtree -f /tmp/demo.mtree -p ./
echo $?


touch $0 # SCANDALOUS
gomtree -f /tmp/demo.mtree -p ./

Tar Archive Support

Verify at rest Filesystems

go get -u github.com/vbatts/go-mtree/cmd/gomtree
gomtree -c -p ./ -K sha256digest | tee /tmp/demo.mtree

gomtree -f /tmp/demo.mtree -p ./
echo $?

read

touch $0 # SCANDALOUS
gomtree -f /tmp/demo.mtree -p ./

Directory Path

Verify at rest Filesystems

mtree -c -p ./ -K sha256digest | tee /tmp/demo.mtree

mtree -f /tmp/demo.mtree -p ./
echo $?

read

touch $0 # SCANDALOUS
mtree -f /tmp/demo.mtree -p ./

BSD mtree || mtree-port

Verify at rest Filesystems

#!/usr/bin/env python

import libarchive

with libarchive.file_writer('../demo.mtree', 'mtree') as a:
    a.add_files('./')

with packages: libarchive and python-libarchive-c

NOTICE: libarchive uses older mtree format

Verify at rest Filesystems

[root@host /]# bsdtar --format mtree -cf foo.mtree /usr
bsdtar: Removing leading '/' from member names
[root@fa97e1919c44 /]# more foo.mtree
#mtree
./usr gname=root uname=root time=1524747817.0 mode=755 gid=0 uid=0 type=dir
./usr/tmp gname=root uname=root time=1517996467.0 mode=777 gid=0 uid=0 type=link link=../var/tmp
./usr/bin gname=root uname=root time=1537976676.897120032 mode=555 gid=0 uid=0 type=dir
./usr/bin/[ gname=root uname=root time=1521637266.0 mode=555 gid=0 uid=0 type=file size=48
./usr/bin/alias gname=root uname=root time=1521122304.0 mode=755 gid=0 uid=0 type=file size=29
[...]

bsdtar (libarchive)

NOTICE: libarchive uses older mtree format

Call to Action

get familiarized with mtree format

github.com/vbatts/go-mtree

make your implementation compatible

consider your provenance and sharing fs metadata use-cases

github.com/vbatts/talks

Thank You!

VINCENT BATTS

@VBATTS| VBATTS@REDHAT.COM