Observability — Advanced

Write your own eBPF programs in C. Build kernel-level telemetry tools. Deploy instrumented kldloadOS images to cloud providers. This is the deep end.

Writing eBPF programs in C

The BCC tools and bpftrace are great for one-off tracing, but production observability tools need compiled eBPF programs — deterministic performance, no runtime compilation, CO-RE (Compile Once, Run Everywhere) portability.

Prerequisites

# Debian
kpkg install clang llvm libbpf-dev linux-headers-$(uname -r) bpftool

# CentOS/RHEL
kpkg install clang llvm libbpf-devel kernel-devel-$(uname -r) bpftool

Project structure

my-ebpf-tool/
├── Makefile
├── src/
│   ├── common.h           ← shared structs between kernel and userspace
│   ├── my_tool.bpf.c      ← eBPF program (runs in kernel)
│   └── my_tool.c           ← userspace loader (runs in userland)
└── build/                   ← compiled output

Step 1: Define shared types (common.h)

// common.h — shared between kernel and userspace
#ifndef __COMMON_H
#define __COMMON_H

struct event {
    __u32 pid;
    __u32 uid;
    __u64 timestamp_ns;
    __u64 latency_ns;
    char comm[16];
    char filename[256];
};

#endif

Step 2: Write the eBPF program (my_tool.bpf.c)

// my_tool.bpf.c — kernel-space eBPF program
#include "vmlinux.h"
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
#include <bpf/bpf_core_read.h>
#include "common.h"

// Ring buffer for events (replaces perf_buffer — lower overhead)
struct {
    __uint(type, BPF_MAP_TYPE_RINGBUF);
    __uint(max_entries, 256 * 1024);  // 256KB
} events SEC(".maps");

// Hash map to track enter timestamps
struct {
    __uint(type, BPF_MAP_TYPE_HASH);
    __uint(max_entries, 10240);
    __type(key, __u64);     // pid_tgid
    __type(value, __u64);   // timestamp
} start_times SEC(".maps");

// Hook: entering openat() syscall
SEC("tracepoint/syscalls/sys_enter_openat")
int trace_openat_enter(struct trace_event_raw_sys_enter *ctx)
{
    __u64 pid_tgid = bpf_get_current_pid_tgid();
    __u64 ts = bpf_ktime_get_ns();
    bpf_map_update_elem(&start_times, &pid_tgid, &ts, BPF_ANY);
    return 0;
}

// Hook: returning from openat() syscall
SEC("tracepoint/syscalls/sys_exit_openat")
int trace_openat_exit(struct trace_event_raw_sys_exit *ctx)
{
    __u64 pid_tgid = bpf_get_current_pid_tgid();

    __u64 *start_ts = bpf_map_lookup_elem(&start_times, &pid_tgid);
    if (!start_ts)
        return 0;

    __u64 latency = bpf_ktime_get_ns() - *start_ts;
    bpf_map_delete_elem(&start_times, &pid_tgid);

    // Only report slow opens (>1ms)
    if (latency < 1000000)
        return 0;

    struct event *e = bpf_ringbuf_reserve(&events, sizeof(*e), 0);
    if (!e)
        return 0;

    e->pid = pid_tgid >> 32;
    e->uid = bpf_get_current_uid_gid() & 0xFFFFFFFF;
    e->timestamp_ns = bpf_ktime_get_ns();
    e->latency_ns = latency;
    bpf_get_current_comm(&e->comm, sizeof(e->comm));

    bpf_ringbuf_submit(e, 0);
    return 0;
}

char LICENSE[] SEC("license") = "GPL";

Step 3: Write the userspace loader (my_tool.c)

// my_tool.c — userspace program that loads and reads eBPF events
#include <stdio.h>
#include <unistd.h>
#include <signal.h>
#include <bpf/libbpf.h>
#include "my_tool.skel.h"   // auto-generated by bpftool
#include "common.h"

static volatile sig_atomic_t running = 1;

static void sig_handler(int sig) { running = 0; }

static int handle_event(void *ctx, void *data, size_t data_sz)
{
    struct event *e = data;
    printf("%-8d %-6d %-16s latency=%.2fms\n",
           e->pid, e->uid, e->comm,
           (double)e->latency_ns / 1000000.0);
    return 0;
}

int main(int argc, char **argv)
{
    struct my_tool_bpf *skel;
    struct ring_buffer *rb;
    int err;

    signal(SIGINT, sig_handler);
    signal(SIGTERM, sig_handler);

    // Load and verify eBPF program
    skel = my_tool_bpf__open_and_load();
    if (!skel) {
        fprintf(stderr, "Failed to load BPF skeleton\n");
        return 1;
    }

    // Attach to tracepoints
    err = my_tool_bpf__attach(skel);
    if (err) {
        fprintf(stderr, "Failed to attach BPF programs: %d\n", err);
        goto cleanup;
    }

    // Set up ring buffer consumer
    rb = ring_buffer__new(bpf_map__fd(skel->maps.events), handle_event, NULL, NULL);
    if (!rb) {
        fprintf(stderr, "Failed to create ring buffer\n");
        err = 1;
        goto cleanup;
    }

    printf("Tracing slow file opens (>1ms)... Ctrl+C to stop.\n");
    printf("%-8s %-6s %-16s %s\n", "PID", "UID", "COMM", "LATENCY");

    while (running) {
        err = ring_buffer__poll(rb, 100);
        if (err == -EINTR) {
            err = 0;
            break;
        }
        if (err < 0) {
            fprintf(stderr, "Ring buffer poll error: %d\n", err);
            break;
        }
    }

cleanup:
    ring_buffer__free(rb);
    my_tool_bpf__destroy(skel);
    return err < 0 ? 1 : 0;
}

Step 4: Build with the Makefile

# Makefile
CLANG     ?= clang
BPFTOOL   ?= bpftool
ARCH      := $(shell uname -m | sed 's/x86_64/x86/' | sed 's/aarch64/arm64/')
BUILD_DIR := build
SRC_DIR   := src

VMLINUX_H := $(SRC_DIR)/vmlinux.h
BPF_OBJ   := $(BUILD_DIR)/my_tool.bpf.o
SKEL_H    := $(SRC_DIR)/my_tool.skel.h
BINARY    := $(BUILD_DIR)/my_tool

.PHONY: all clean

all: $(BINARY)

$(BUILD_DIR):
    mkdir -p $@

# Generate vmlinux.h from running kernel's BTF
$(VMLINUX_H):
    $(BPFTOOL) btf dump file /sys/kernel/btf/vmlinux format c > $@

# Compile eBPF program
$(BPF_OBJ): $(SRC_DIR)/my_tool.bpf.c $(VMLINUX_H) $(SRC_DIR)/common.h | $(BUILD_DIR)
    $(CLANG) -g -O2 -target bpf -D__TARGET_ARCH_$(ARCH) \
        -I$(SRC_DIR) \
        -c $< -o $@

# Generate skeleton header from compiled BPF object
$(SKEL_H): $(BPF_OBJ)
    $(BPFTOOL) gen skeleton $< > $@

# Compile userspace loader
$(BINARY): $(SRC_DIR)/my_tool.c $(SKEL_H) $(SRC_DIR)/common.h | $(BUILD_DIR)
    $(CLANG) -g -O2 -I$(SRC_DIR) \
        $< -lbpf -lelf -lz -o $@

clean:
    rm -rf $(BUILD_DIR) $(VMLINUX_H) $(SKEL_H)

make clean && make all
sudo ./build/my_tool

How CO-RE works

The vmlinux.h generated from your kernel’s BTF (BPF Type Format) contains all kernel struct definitions. The BPF CO-RE relocations in the compiled .bpf.o allow the same binary to run on different kernel versions — the loader adjusts struct field offsets at load time.

This means you can compile on one kldloadOS system and deploy the binary to any other kldloadOS system, regardless of kernel version.

Building a production eBPF service

Take the pattern above and add:

Prometheus metrics export

// In your userspace loader, add a simple HTTP metrics endpoint
// Or write the events to a Unix socket and have a Python sidecar export them

// Simple approach: write events to a ring buffer → Python consumer → prometheus_client

Better approach — write the eBPF C program for the kernel-space part and use Python BCC for the userspace consumer (like socket_snoop and latency_snoop do). This gives you:

• C performance in the kernel
• Python flexibility in userspace
• prometheus_client for metrics
• JSON output for log aggregation

Package as a systemd service

cat > /etc/systemd/system/my-ebpf-tool.service << 'EOF'
[Unit]
Description=Custom eBPF telemetry
After=network.target

[Service]
Type=simple
ExecStart=/usr/local/bin/my_tool
Restart=always
RestartSec=5

# Security hardening
CapabilityBoundingSet=CAP_BPF CAP_PERFMON CAP_NET_ADMIN CAP_SYS_RESOURCE
AmbientCapabilities=CAP_BPF CAP_PERFMON CAP_NET_ADMIN CAP_SYS_RESOURCE

[Install]
WantedBy=multi-user.target
EOF

cp build/my_tool /usr/local/bin/
systemctl daemon-reload
systemctl enable --now my-ebpf-tool

Deploying instrumented images to AWS

Build a kldloadOS system with your observability stack baked in, then push it to AWS as an AMI.

Step 1: Build and configure locally

# Build the ISO
PROFILE=server ./deploy.sh build

# Install to a local VM
./deploy.sh kvm-deploy

# SSH in and set up your stack:
# - Install socket_snoop + latency_snoop as systemd services
# - Install Prometheus node_exporter
# - Install your custom eBPF tools
# - Configure firewall rules
# - Shut down

Step 2: Export to raw image

# Inside the VM
kexport raw

Step 3: Upload to AWS

# Convert qcow2 → raw if needed
qemu-img convert -f qcow2 -O raw kldload-export-*.qcow2 kldload.raw

# Upload to S3
aws s3 cp kldload.raw s3://my-images/kldload-server.raw

# Import as AMI
aws ec2 import-image \
  --disk-containers "Format=RAW,UserBucket={S3Bucket=my-images,S3Key=kldload-server.raw}" \
  --description "kldloadOS server with observability stack" \
  --boot-mode uefi

# Check import progress
aws ec2 describe-import-image-tasks

Step 4: Launch

aws ec2 run-instances \
  --image-id ami-xxxxxxxxxxxx \
  --instance-type t3.medium \
  --key-name my-key \
  --security-group-ids sg-xxxx \
  --subnet-id subnet-xxxx \
  --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=kldload-monitored}]'

Or use the deploy-vm.sh helper:

cd /opt/linux-tools/aws/deploy-vm
AWS_REGION=us-west-2 INSTANCE_TYPE=t3.medium ./deploy-vm.sh

Step 5: Verify from inside the instance

# Check instance metadata
/opt/linux-tools/aws/meta-scrape/meta-scrape.py

Shows: instance ID, region, availability zone, AMI ID, instance type, public/private IPs, IAM role, security groups.

# Verify observability stack
kst                                        # health check
systemctl status socket-snoop              # socket events
systemctl status latency-snoop             # latency metrics
curl -s localhost:9900/metrics | head -20  # Prometheus metrics
curl -s localhost:9100/metrics | head -20  # node_exporter

Deploying to Azure

# Export as VHD
kexport vhd

# Upload
az storage blob upload \
  --account-name myaccount \
  --container-name images \
  --name kldload-server.vhd \
  --type page \
  --file kldload-export-*.vhd

# Create image
az image create \
  --resource-group mygroup \
  --name kldloadOS-server \
  --os-type Linux \
  --source https://myaccount.blob.core.windows.net/images/kldload-server.vhd

# Launch
az vm create \
  --resource-group mygroup \
  --name kldload-monitored \
  --image kldloadOS-server \
  --size Standard_B2ms \
  --admin-username admin \
  --ssh-key-values ~/.ssh/id_rsa.pub

Building an observability golden image

Automate the entire stack into a reproducible golden image:

#!/bin/bash
# build-observability-image.sh
# Creates a kldloadOS server with full observability stack

set -euo pipefail

# 1. Build the ISO
PROFILE=server ./deploy.sh build

# 2. Create a VM and install
ISO=$(ls -t live-build/output/*.iso | head -1)
virt-install \
  --name obs-golden \
  --ram 4096 --vcpus 4 \
  --disk size=40,format=qcow2 \
  --cdrom "$ISO" \
  --os-variant centos-stream9 \
  --boot uefi \
  --noautoconsole --wait

# 3. Copy setup script into the VM and run it
virt-customize -d obs-golden \
  --run-command 'kpkg install bcc-tools bpftrace perf' \
  --mkdir /opt/linux-tools \
  --copy-in /opt/linux-tools/debian/monitoring:/opt/linux-tools/debian/ \
  --run-command 'cd /opt/linux-tools/debian/monitoring && make setup && make deps' \
  --copy-in /etc/systemd/system/socket-snoop.service:/etc/systemd/system/ \
  --copy-in /etc/systemd/system/latency-snoop.service:/etc/systemd/system/ \
  --run-command 'systemctl enable socket-snoop latency-snoop' \
  --hostname obs-golden

# 4. Generalize
virt-sysprep -d obs-golden --operations defaults,-ssh-userdir --hostname localhost

# 5. Snapshot for cloning
virsh shutdown obs-golden 2>/dev/null || true
echo "Golden image ready at /var/lib/libvirt/images/obs-golden.qcow2"
echo "Clone with: qemu-img create -f qcow2 -b obs-golden.qcow2 -F qcow2 new-node.qcow2"

Now every new node you clone from this image comes up with the full observability stack running.

Full stack architecture

┌──────────────────────────────────────────────────────────────┐
│                     kldloadOS Node                           │
│                                                              │
│  ┌─────────────┐  ┌──────────────┐  ┌────────────────────┐  │
│  │ socket_snoop │  │latency_snoop │  │ custom eBPF tools  │  │
│  │  (events)    │  │  (metrics)   │  │   (your code)      │  │
│  └──────┬───────┘  └──────┬───────┘  └─────────┬──────────┘  │
│         │                 │                     │             │
│         ▼                 ▼                     ▼             │
│  /var/log/socket    :9900/metrics         :9901/metrics      │
│  _monitor.log       (Prometheus)          (Prometheus)       │
│         │                 │                     │             │
│  ┌──────▼───────┐  ┌─────▼─────────────────────▼──────────┐  │
│  │   LogHog     │  │           Prometheus                  │  │
│  │   (forensics)│  │  scrape all :9xxx endpoints           │  │
│  └──────────────┘  └──────────────┬────────────────────────┘  │
│                                   │                           │
│                            ┌──────▼──────┐                    │
│                            │   Grafana    │                   │
│                            │  dashboards  │                   │
│                            └─────────────┘                    │
│                                                              │
│  ┌────────────┐  ┌────────────┐  ┌────────────────────────┐  │
│  │    kst      │  │diagnostics │  │   node_exporter        │  │
│  │ (one-shot)  │  │  (report)  │  │  (system metrics :9100)│  │
│  └────────────┘  └────────────┘  └────────────────────────┘  │
└──────────────────────────────────────────────────────────────┘

Beginner tools (bottom row): kst for quick checks, diagnostics.sh for full reports, node_exporter for basic system metrics.

Intermediate tools (middle): socket_snoop for event streams, latency_snoop for Prometheus metrics, LogHog for log forensics.

Advanced tools (top): Your custom eBPF programs, compiled with CO-RE, exporting to Prometheus, visualized in Grafana.

Reference: linux-tools repo

The observability tools referenced in this guide come from github.com/unixbox-net/linux-tools: