# Passive FIlesystem VAlidation

bit.ly/asg2018-vbatts-pfv

Vincent Batts @vbatts

    $> finger $(whoami)
    Login: vbatts                           Name: Vincent Batts
    Directory: /home/vbatts                 Shell: /bin/bash
    Such mail.
    Plan:
    OHMAN
    $> id -Gn
    devel opencontainers docker appc redhat golang slackware

- Packaging
- Content Addressibility
- Compression!
- Reproducible Archives
- Verify at rest filesystems

## Agenda

## Packaging

tar archives

Slackware packages (tar(1) archives)

Debian *.deb (ar(1) archive of tar(1) archives)

Red Hat *.rpm (custom key/value binary and cpio(1))

Java *.jar and *.war (zip(1) archive)

Ruby *.gem (tar(1) archive of tar(1) archives)

Container Images (tar(1) archives)

## Content Addressibility

Opaque Object storage

changed object = new object

cryptographic assurance

## compression!

inflate/deflate (RFC1951)

same objects, but variation in compression

Gzip (RFC1952)

`gzip` vs Golang `compress/gzip` vs Zlib

ideally compress for transfer and storage, but not for identity

## compression!

    #!/bin/sh
    dd if=/dev/urandom of=rando.img bs=1M count=2
    cat rando.img | gzip -n > rando.img.gz
    cat rando.img | gzip -n -9 > rando.img.9.gz
    cat rando.img | xz > rando.img.xz
    cat rando.img | xz -9 > rando.img.9.xz
    sha1sum rando.img* > SHA1

    cat rando.img | gzip -n > rando.img.gz
    cat rando.img | gzip -n -9 > rando.img.9.gz
    cat rando.img | xz > rando.img.xz
    cat rando.img | xz -9 > rando.img.9.xz
    sha1sum -c ./SHA1

## compression!

    #!/usr/bin/env ruby

    require 'zlib'
    include Zlib

    input = File.open(ARGV.first)
    GzipWriter.open(ARGV.first + '.gz', DEFAULT_COMPRESSION, HUFFMAN_ONLY) do |gz|
      gz.write(IO.binread(input))
    end
    input.flush()
    input.close()

## compression!

    package main

    import (
            "compress/gzip"
            "io"
            "os"
    )

    func main() {
            input, err := os.Open(os.Args[1])
            if err != nil {
                    println(err.Error())
                    os.Exit(1)
            }
            output, err := os.Create(os.Args[1] + ".gz")
            if err != nil {
                    println(err.Error())
                    os.Exit(1)
            }
            gz := gzip.NewWriter(output)
            if _, err := io.Copy(gz, input); err != nil {
                    println(err.Error())
                    os.Exit(1)
            }
    }

## Verify at rest Filesystems

Regardless of transport, ensure resulting filesystem

(*.tar archive, rsync, bittorrent, IPFS, etc)

`rpm -qV <package>` functionality

Future hopes could be IMA/EVM

Passive validation of directory hierarchies

BSD mtree(8)

dm-verity in select use-cases

## Verify at rest Filesystems

FreeBSD mtree(8)

mtree-port (for linux)

go-mtree (golang cli and library)

libarchive-formats(5) (so `bsdtar`)

casync mtree

umoci unpack

## Verify at rest Filesystems

    [root@host /]# gomtree -c -K sha256 -p /usr/ | head -30
    #          user: root
    #       machine: host
    #          tree: /usr
    #          date: Wed Sep 26 16:07:53 2018
    #      keywords: size,type,uid,gid,mode,link,nlink,time,sha256digest

    # .
    /set type=file nlink=1 mode=0664 uid=0 gid=0
    . size=100 type=dir mode=0755 time=1524747817.000000000
        tmp size=10 type=link mode=0777 link=../var/tmp time=1517996467.000000000

    # bin
    bin size=5268 type=dir mode=0555 time=1537977678.074646319
        \133 size=48 mode=0555 time=1521637266.000000000 sha256digest=afd97bbd643bfe1473794af167cd5c6f44fe449681033e3584b40b836f624b4b
        alias size=29 mode=0755 time=1521122304.000000000 sha256digest=c9e358c5012c2cf9171ec4f7692ac3a1cbc280617b610d77d813653a1d0dfeb8
        applydeltarpm size=72752 mode=0755 time=1517985721.000000000 sha256digest=359f076a0a259bdab3d8139546319efe076fe49733c44aae1a823872e51451eb
        arch size=51 mode=0555 time=1521637267.000000000 sha256digest=209bae4071910ef54b4a3bd302059bf7e00870d8bacffcd7c5489425f37ed16f

gomtree

## Verify at rest Filesystems

    [root@host /]# casync mtree /usr/
    . type=dir mode=0755 uid=0 gid=0 time=1524747817.000000000
    bin type=dir mode=0555 uid=0 gid=0 time=1537977212.955190222
    bin/[ type=file mode=0555 size=48 uid=0 gid=0 time=1521637266.000000000 sha512256digest=f02fe283c47d25e8273a4ce6297eef3e9f2300f597207fa6b82fc12694bd9771
    bin/alias type=file mode=0755 size=29 uid=0 gid=0 time=1521122304.000000000 sha512256digest=c526d0d764732aa40fa866033bc49882f1e575ff89f969261eb354945db57d75
    bin/applydeltarpm type=file mode=0755 size=72752 uid=0 gid=0 time=1517985721.000000000 sha512256digest=794cd98d3aac69cd3a2ab0e6820e0070555fdbc79de906a79addb98b919baa47
    bin/arch type=file mode=0555 size=51 uid=0 gid=0 time=1521637267.000000000 sha512256digest=5516a6b4ee54e9f5fa7d8842781bcb6b3799e2dfa581ce1941c1c9f06a22d648
    bin/awk type=link mode=0777 link=gawk uid=0 gid=0 time=1519649348.000000000
    bin/b2sum type=file mode=0555 size=52 uid=0 gid=0 time=1521637266.000000000 sha512256digest=423c318bb3cad9826bf3b932baccb66baf2c7f0d55cb0d7d10f41d7e4ebb8fe6
    bin/base32 type=file mode=0555 size=53 uid=0 gid=0 time=1521637266.000000000 sha512256digest=45b43efd608aca559aa0dea075a46a554f7e7fcc3534ea2c7f8edc344a342ead
    bin/base64 type=file mode=0555 size=53 uid=0 gid=0 time=1521637266.000000000 sha512256digest=d63d00e263f95f09a27ddd4fc6a00b76835e5371af0a2c9a32e3929305810536
    [...]

casync mtree

## Verify at rest Filesystems

    > skopeo copy docker://docker.io/busybox:latest oci:busybox:latest
    > umoci unpack --image ./busybox:latest busybox-bundle
    > cat busybox-bundle/sha256_9b9b48e2d92691f344ad1701e7df04f89dd2041f2f05cccfec39af8ac3a62d25.mtree
    #          user: root
    #       machine: host
    #          tree: busybox-bundle/rootfs
    #          date: Wed Sep 26 16:21:46 2018
    #      keywords: size,type,uid,gid,mode,link,nlink,tar_time,sha256digest,xattr

    # .
    /set type=file nlink=1 mode=0664 uid=0 gid=0
    . size=52 type=dir mode=0755 tar_time=0.000000000

    # bin
    bin size=4638 type=dir mode=0755 tar_time=1533068407.000000000
        \133 size=1083720 mode=0755 nlink=393 tar_time=1533068407.000000000 sha256digest=ee8001feee768df41ff9a2b704b4a027d7ed76f78f09648a5b7ad9f1d7d45640
        \133\133 size=1083720 mode=0755 nlink=393 tar_time=1533068407.000000000 sha256digest=ee8001feee768df41ff9a2b704b4a027d7ed76f78f09648a5b7ad9f1d7d45640
        acpid size=1083720 mode=0755 nlink=393 tar_time=1533068407.000000000 sha256digest=ee8001feee768df41ff9a2b704b4a027d7ed76f78f09648a5b7ad9f1d7d45640
        add-shell size=1083720 mode=0755 nlink=393 tar_time=1533068407.000000000 sha256digest=ee8001feee768df41ff9a2b704b4a027d7ed76f78f09648a5b7ad9f1d7d45640
        addgroup size=1083720 mode=0755 nlink=393 tar_time=1533068407.000000000 sha256digest=ee8001feee768df41ff9a2b704b4a027d7ed76f78f09648a5b7ad9f1d7d45640
        adduser size=1083720 mode=0755 nlink=393 tar_time=1533068407.000000000 sha256digest=ee8001feee768df41ff9a2b704b4a027d7ed76f78f09648a5b7ad9f1d7d45640
        adjtimex size=1083720 mode=0755 nlink=393 tar_time=1533068407.000000000 sha256digest=ee8001feee768df41ff9a2b704b4a027d7ed76f78f09648a5b7ad9f1d7d45640
    [...]

umoci unpack

## Verify at rest Filesystems

    tar cf /tmp/demo.tar .
    gomtree -c -T /tmp/demo.tar -K sha256digest | tee /tmp/demo.mtree

    gomtree -f /tmp/demo.mtree -T /tmp/demo.tar
    echo $?

    read

    gomtree -f /tmp/demo.mtree -p ./
    echo $?

    touch $0 # SCANDALOUS
    gomtree -f /tmp/demo.mtree -p ./

Tar Archive Support

## Verify at rest Filesystems

    go get -u github.com/vbatts/go-mtree/cmd/gomtree
    gomtree -c -p ./ -K sha256digest | tee /tmp/demo.mtree

    gomtree -f /tmp/demo.mtree -p ./
    echo $?

    read

    touch $0 # SCANDALOUS
    gomtree -f /tmp/demo.mtree -p ./

Directory Path

## Verify at rest Filesystems

    mtree -c -p ./ -K sha256digest | tee /tmp/demo.mtree

    mtree -f /tmp/demo.mtree -p ./
    echo $?

    read

    touch $0 # SCANDALOUS
    mtree -f /tmp/demo.mtree -p ./

BSD mtree || mtree-port

## Verify at rest Filesystems

    #!/usr/bin/env python

    import libarchive

    with libarchive.file_writer('../demo.mtree', 'mtree') as a:
        a.add_files('./')

with packages: libarchive and python-libarchive-c

NOTICE: libarchive uses older mtree format

## Verify at rest Filesystems

    [root@host /]# bsdtar --format mtree -cf foo.mtree /usr
    bsdtar: Removing leading '/' from member names
    [root@fa97e1919c44 /]# more foo.mtree
    #mtree
    ./usr gname=root uname=root time=1524747817.0 mode=755 gid=0 uid=0 type=dir
    ./usr/tmp gname=root uname=root time=1517996467.0 mode=777 gid=0 uid=0 type=link link=../var/tmp
    ./usr/bin gname=root uname=root time=1537976676.897120032 mode=555 gid=0 uid=0 type=dir
    ./usr/bin/[ gname=root uname=root time=1521637266.0 mode=555 gid=0 uid=0 type=file size=48
    ./usr/bin/alias gname=root uname=root time=1521122304.0 mode=755 gid=0 uid=0 type=file size=29
    [...]

bsdtar (libarchive)

NOTICE: libarchive uses older mtree format

## Call to Action

get familiarized with mtree format

github.com/vbatts/go-mtree

make your implementation compatible

consider your provenance and sharing fs metadata use-cases

### github.com/vbatts/talks

### Thank You!

## VINCENT BATTS

## @VBATTS| VBATTS@REDHAT.COM