Visualize

We can build some GraphViz diagrams that help up to understand better some configurations on the server. This can be done automatically from bash scripts.

First, let’s install GraphViz:

apt install graphviz

1. Port forwarding

This script builds a graph that shows port forwarding on the server. Usually the open ports:

go to the host (the ports on the public zone of firewalld)
are forwarded to an Incus container
are forwarded to a Docker container

Script: visualize/gv-ports.sh

#!/bin/bash

main() {
    local file_gv=${1:-/dev/stdout}

    # get container name from ip
    declare -A container_name
    for container in $(get_containers); do
        ip=$(get_ip $container)
        container_name[$ip]=$container
    done

    print_gv_file $file_gv
}

get_containers() {
    incus ls -f csv -c ns \
        | grep RUNNING \
        | sed 's/,RUNNING//'
}

get_ip() {
    local container=$1
    if [[ -z $container ]]; then
        ip route get 8.8.8.8 | head -1 | sed 's/.*src //' | cut -d' ' -f1
    else
        incus exec $container -- \
              ip route get 8.8.8.8 | head -1 | sed 's/.*src //' | cut -d' ' -f1
    fi
}

print_gv_file() {
    local file_gv=${1:-/dev/stdout}
    cat <<EOF > $file_gv
digraph {
    graph [
        rankdir = LR
        ranksep = 1.5
        concentrate = true
        fontname = system
        fontcolor = brown
    ]

    // all the ports that are accessable in the server
    subgraph ports {
        graph [
            cluster = true
            //label = "Ports"
            style = "rounded, dotted"
        ]
        node [
            fillcolor = lightyellow
            style = "filled"
            fontname = system
            fontcolor = navy
        ]

        // ports that are forwarded to the host itself
        // actually they are the ports that are open in the firewall
        subgraph host_ports {
            graph [
                cluster = true
                style = "rounded"
                color = darkgreen
                bgcolor = lightgreen
                label = "firewall-cmd --list-ports"
            ]

            $(host_port_nodes)
        }

        // ports that are forwarded to the docker containers
        subgraph docker_ports {
            graph [
                cluster = true
                label = "docker ports"
                style = "rounded"
                bgcolor = lightblue
                color = steelblue
            ]

            $(docker_port_nodes)
        }

        // ports that are forwarded to the incus containers
        // with the command: 'incus network forward ...'
        subgraph incus_ports {
            graph [
                cluster = true
                label = "incus network forward"
                style = "rounded"
                bgcolor = pink
                color = orangered
            ]

            $(incus_port_nodes)
        }
    }

    Host [
        label = "Host: $(hostname -f)"
        margin = 0.2
        shape = box
        peripheries = 2
        style = "filled, rounded"
        color = red
        fillcolor = orange
        fontname = system
        fontcolor = navy
    ]

    // docker containers
    subgraph docker_containers {
        graph [
            cluster = true
            label = "Docker"
            style = "rounded"
        ]
        node [
            shape = record
            style = "filled, rounded"
            color = blue
            fillcolor = deepskyblue
            fontname = system
            fontcolor = navy
        ]

        $(docker_container_nodes)
    }
    // incus containers
    subgraph incus_containers {
        graph [
            cluster = true
            label = "Incus"
            style = "rounded"
        ]
        node [
            shape = record
            style = "filled, rounded"
            color = orange
            fillcolor = yellow
            fontname = system
            fontcolor = navy
        ]

        $(incus_container_nodes)
    }

    edge [
        dir = back
        arrowtail = odiamondinvempty
    ]

    $(host_edges)
    $(docker_edges)
    $(incus_edges)

    graph [
        labelloc = t
        //labeljust = r
        fontcolor = red
        label = "Port forwarding"
    ]
}
EOF
}

print_port_node() {
    local protocol=$1
    local port=$2
    cat <<EOF
            "$protocol/$port" [ label = <$protocol <br/> <b>$port</b>> ];
EOF
}

host_port_nodes() {
    echo
    local ports=$(firewall-cmd --list-ports --zone=public)
    for p in $ports; do
        protocol=$(echo $p | cut -d'/' -f2)
        port=$(echo $p | cut -d'/' -f1)
        print_port_node $protocol $port
    done
}

host_edges() {
    echo
    local ports=$(firewall-cmd --list-ports --zone=public)
    for p in $ports; do
        protocol=$(echo $p | cut -d'/' -f2)
        port=$(echo $p | cut -d'/' -f1)
        cat <<EOF
    "$protocol/$port" -> Host;
EOF
    done
}

docker_port_nodes() {
    echo
    docker container ls -a --format "{{.Names}} {{.Ports}}" \
        | grep 0.0.0.0 \
        | sed -e 's/,//g' \
        | tr ' ' "\n" \
        | grep 0.0.0.0 \
        | cut -d: -f2 \
        | \
        while read line; do
            port=$(echo $line | cut -d'-' -f1)
            protocol=$(echo $line | cut -d'/' -f2)
            print_port_node $protocol $port
        done
}

docker_container_nodes() {
    echo
    docker container ls -a --format "{{.Names}} {{.Ports}}" \
        | grep 0.0.0.0 \
        | cut -d' '  -f1 \
        | \
        while read name; do
            echo "        \"$name\";"
        done
}

docker_edges() {
    echo
    docker container ls -a --format "{{.Names}} {{.Ports}}" \
        | grep 0.0.0.0 \
        | sed -e 's/,//g' \
        | \
        while read line; do
            container=$(echo $line | cut -d' ' -f1)
            echo $line | tr ' ' "\n" | grep 0.0.0.0 | cut -d: -f2 | \
                while read line1; do
                    port=$(echo $line1 | cut -d'-' -f1)
                    protocol=$(echo $line1 | cut -d'/' -f2)
                    target=$(echo $line1 | cut -d'>' -f2 | cut -d'/' -f1)
                    [[ $target == $port ]] && target=''
                    cat <<EOF
    "$protocol/$port" -> "$container" [ label = "$target" ];
EOF
                done
        done
}

incus_port_nodes() {
    echo
    get_forward_lines | \
        while read line; do
            eval "$line"
            target_port=${target_port:-$listen_port}
            #echo $description $protocol $listen_port $target_port $target_address
            print_port_node $protocol $listen_port
        done
}

get_forward_lines() {
    local listen_address=$(get_ip)
    incus network forward show incusbr0 $listen_address \
        | sed -e '/^description:/d' \
              -e '/^config:/d' \
              -e '/^ports:/d' \
              -e '/^listen_address:/d' \
              -e '/^location:/d' \
        | tr -d '\n\r' \
        | sed -e 's/- description:/\ndescription:/g' \
        | sed -e 1d \
        | sed -e 's/: /=/g'
}

incus_container_nodes() {
    echo
    for container in $(get_containers); do
        ip=$(get_ip $container)
        cat <<EOF
        $container [ label = "$container|<ip> $ip" ];
EOF
    done
}

incus_edges() {
    echo
    get_forward_lines | \
        while read line; do
            eval "$line"
            #echo $description $protocol $listen_port $target_port $target_address
            container=${container_name[$target_address]}
            cat <<EOF
    "$protocol/$listen_port" -> $container:ip [ label = "$target_port" ];
EOF
        done
}

### call main
main "$@"

To build the graph do:

./gv-ports.sh ports.gv
dot -Tpng ports.gv -o ports.png

2. Web domains

The web domains hosted on the server are handled by sniproxy and revproxy. The first one, sniproxy, redirects some of the domains to Incus containers, and the rest to revproxy (which is based on nginx). Then, revproxy redirects these domains to docker dontainers.

Script: visualize/gv-domains.sh

#!/bin/bash

main() {
    local file_gv=${1:-/dev/stdout}

    # get container name from ip
    declare -A container_name
    for container in $(get_containers); do
        ip=$(get_ip $container)
        container_name[$ip]=$container
    done

    print_gv_file $file_gv
}

get_containers() {
    incus ls -f csv -c ns \
        | grep RUNNING \
        | sed 's/,RUNNING//'
}

get_ip() {
    local container=$1
    if [[ -z $container ]]; then
        ip route get 8.8.8.8 | head -1 | sed 's/.*src //' | cut -d' ' -f1
    else
        incus exec $container -- \
              ip route get 8.8.8.8 | head -1 | sed 's/.*src //' | cut -d' ' -f1
    fi
}

print_gv_file() {
    local file_gv=${1:-/dev/stdout}
    cat <<EOF > $file_gv
digraph {
    graph [
        rankdir = LR
        ranksep = 1.5
        concentrate = true
        fontname = system
        fontcolor = brown
    ]

    node [
        shape = box
    ]

    // domain entries of sniproxy
    subgraph sniproxy_domains {
        graph [
            cluster = true
            label = sniproxy
            color = darkgreen
            bgcolor = lightgreen
            style = "rounded, dotted"
        ]

        node [
            fillcolor = lightyellow
            style = "filled"
            fontname = system
            fontcolor = navy
        ]

        $(sniproxy_domains)
    }

    // domain entries of revproxy
    subgraph revproxy_domains {
        graph [
            cluster = true
            color = darkgreen
            bgcolor = lightgreen
            style = "rounded, dotted"
        ]

        node [
            fillcolor = lightyellow
            style = "filled"
            fontname = system
            fontcolor = navy
        ]

        $(revproxy_domains)
    }

    revproxy [
        label = "revproxy"
        margin = 0.2
        shape = box
        peripheries = 2
        style = "filled, rounded"
        color = blue
        fillcolor = deepskyblue
        fontname = system
        fontcolor = navy
    ]

    node [
        shape = record
        style = "filled, rounded"
        color = orange
        fillcolor = yellow
        fontname = system
        fontcolor = navy
    ]

    $(incus_container_nodes)

    edge [
        arrowhead = none
    ]

    $(sniproxy_edges)
    $(revproxy_edges)

    graph [
        labelloc = t
        //labeljust = r
        fontcolor = red
        label = "Domains"
    ]
}
EOF
}

sniproxy_domains() {
    echo
    cat /var/ds/sniproxy/etc/sniproxy.conf \
        | sed -e '1,/table/d' -e '/}/d' -e '/# /d' -e '/^ *$/d' \
        | \
        while read line; do
            domain=$(echo "$line" | cut -d' ' -f1)
            echo "        \"$domain\";"
        done
}

revproxy_domains() {
    echo
    for domain in $(ds @revproxy domains-ls); do
        echo "        \"$domain\";"
    done
}

incus_container_nodes() {
    echo
    for container in $(get_containers); do
        ip=$(get_ip $container)
        cat <<EOF
    $container [ label = "$container|<ip> $ip" ];
EOF
    done
}

sniproxy_edges() {
    echo
    cat /var/ds/sniproxy/etc/sniproxy.conf \
        | sed -e '1,/table/d' -e '/}/d' -e '/# /d' -e '/^ *$/d' \
        | \
        while read line; do
            domain=$(echo "$line" | xargs | cut -d' ' -f1)
            target=$(echo "$line" | xargs | cut -d' ' -f2)
            container=${container_name[$target]}
            [[ -z $container ]] && container=$target
            echo "    \"$domain\" -> \"$container\";"
        done
}

revproxy_edges() {
    echo
    for domain in $(ds @revproxy domains-ls); do
        echo "    revproxy -> \"$domain\";"
    done
}

### call main
main "$@"

To build the graph do:

./gv-domains.sh domains.gv
dot -Tpng domains.gv -o domains.png