diff --git a/agent/go.mod b/agent/go.mod index 48b53e8de..6731cb356 100644 --- a/agent/go.mod +++ b/agent/go.mod @@ -54,6 +54,7 @@ require ( github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4 // indirect github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 // indirect github.com/chanced/dynamic v0.0.0-20211210164248-f8fadb1d735b // indirect + github.com/cilium/ebpf v0.8.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect diff --git a/agent/go.sum b/agent/go.sum index 452dbe7c3..7e92431ba 100644 --- a/agent/go.sum +++ b/agent/go.sum @@ -137,6 +137,8 @@ github.com/chanced/openapi v0.0.7/go.mod h1:SxE2VMLPw+T7Vq8nwbVVhDF2PigvRF4n5Xyq github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cilium/ebpf v0.8.0 h1:2V6KSg3FRADVU2BMIRemZ0hV+9OM+aAHhZDjQyjJTAs= +github.com/cilium/ebpf v0.8.0/go.mod h1:f5zLIM0FSNuAkSyLAN7X+Hy6yznlF1mNiWUMfxMtrgk= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -210,8 +212,9 @@ github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYF github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= +github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss= +github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= @@ -1158,6 +1161,7 @@ golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/agent/pkg/controllers/config_controller.go b/agent/pkg/controllers/config_controller.go index d4af64b76..21fae044c 100644 --- a/agent/pkg/controllers/config_controller.go +++ b/agent/pkg/controllers/config_controller.go @@ -57,7 +57,7 @@ func PostTapConfig(c *gin.Context) { ctx, cancel := context.WithCancel(context.Background()) - if _, err := startMizuTapperSyncer(ctx, kubernetesProvider, tappedNamespaces, *podRegex, []string{}, tapApi.TrafficFilteringOptions{}, false); err != nil { + if _, err := startMizuTapperSyncer(ctx, kubernetesProvider, tappedNamespaces, *podRegex, []string{}, tapApi.TrafficFilteringOptions{}, false, false); err != nil { c.JSON(http.StatusInternalServerError, err) cancel() return @@ -100,7 +100,7 @@ func GetTapConfig(c *gin.Context) { c.JSON(http.StatusOK, tapConfigToReturn) } -func startMizuTapperSyncer(ctx context.Context, provider *kubernetes.Provider, targetNamespaces []string, podFilterRegex regexp.Regexp, ignoredUserAgents []string, mizuApiFilteringOptions tapApi.TrafficFilteringOptions, serviceMesh bool) (*kubernetes.MizuTapperSyncer, error) { +func startMizuTapperSyncer(ctx context.Context, provider *kubernetes.Provider, targetNamespaces []string, podFilterRegex regexp.Regexp, ignoredUserAgents []string, mizuApiFilteringOptions tapApi.TrafficFilteringOptions, serviceMesh bool, tls bool) (*kubernetes.MizuTapperSyncer, error) { tapperSyncer, err := kubernetes.CreateAndStartMizuTapperSyncer(ctx, provider, kubernetes.TapperSyncerConfig{ TargetNamespaces: targetNamespaces, PodFilterRegex: podFilterRegex, @@ -113,6 +113,7 @@ func startMizuTapperSyncer(ctx context.Context, provider *kubernetes.Provider, t MizuApiFilteringOptions: mizuApiFilteringOptions, MizuServiceAccountExists: true, //assume service account exists since install mode will not function without it anyway ServiceMesh: serviceMesh, + Tls: tls, }, time.Now()) if err != nil { diff --git a/cli/cmd/tap.go b/cli/cmd/tap.go index faa7316ae..35e79e142 100644 --- a/cli/cmd/tap.go +++ b/cli/cmd/tap.go @@ -120,4 +120,5 @@ func init() { tapCmd.Flags().String(configStructs.EnforcePolicyFile, defaultTapConfig.EnforcePolicyFile, "Yaml file path with policy rules") tapCmd.Flags().String(configStructs.ContractFile, defaultTapConfig.ContractFile, "OAS/Swagger file to validate to monitor the contracts") tapCmd.Flags().Bool(configStructs.ServiceMeshName, defaultTapConfig.ServiceMesh, "Record decrypted traffic if the cluster is configured with a service mesh and with mtls") + tapCmd.Flags().Bool(configStructs.TlsName, defaultTapConfig.Tls, "Record tls traffic") } diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go index 4e8383446..33b17b6ed 100644 --- a/cli/cmd/tapRunner.go +++ b/cli/cmd/tapRunner.go @@ -201,6 +201,7 @@ func startTapperSyncer(ctx context.Context, cancel context.CancelFunc, provider MizuApiFilteringOptions: mizuApiFilteringOptions, MizuServiceAccountExists: state.mizuServiceAccountExists, ServiceMesh: config.Config.Tap.ServiceMesh, + Tls: config.Config.Tap.Tls, }, startTime) if err != nil { diff --git a/cli/config/configStructs/tapConfig.go b/cli/config/configStructs/tapConfig.go index dad2495d1..1dd747745 100644 --- a/cli/config/configStructs/tapConfig.go +++ b/cli/config/configStructs/tapConfig.go @@ -23,6 +23,7 @@ const ( EnforcePolicyFile = "traffic-validation-file" ContractFile = "contract" ServiceMeshName = "service-mesh" + TlsName = "tls" ) type TapConfig struct { @@ -45,6 +46,7 @@ type TapConfig struct { ApiServerResources shared.Resources `yaml:"api-server-resources"` TapperResources shared.Resources `yaml:"tapper-resources"` ServiceMesh bool `yaml:"service-mesh" default:"false"` + Tls bool `yaml:"tls" default:"false"` } func (config *TapConfig) PodRegex() *regexp.Regexp { diff --git a/shared/kubernetes/mizuTapperSyncer.go b/shared/kubernetes/mizuTapperSyncer.go index afe96df61..50e6f1250 100644 --- a/shared/kubernetes/mizuTapperSyncer.go +++ b/shared/kubernetes/mizuTapperSyncer.go @@ -46,6 +46,7 @@ type TapperSyncerConfig struct { MizuApiFilteringOptions api.TrafficFilteringOptions MizuServiceAccountExists bool ServiceMesh bool + Tls bool } func CreateAndStartMizuTapperSyncer(ctx context.Context, kubernetesProvider *Provider, config TapperSyncerConfig, startTime time.Time) (*MizuTapperSyncer, error) { @@ -324,6 +325,7 @@ func (tapperSyncer *MizuTapperSyncer) updateMizuTappers() error { tapperSyncer.config.MizuApiFilteringOptions, tapperSyncer.config.LogLevel, tapperSyncer.config.ServiceMesh, + tapperSyncer.config.Tls, ); err != nil { return err } diff --git a/shared/kubernetes/provider.go b/shared/kubernetes/provider.go index 3ce71a56b..027091bb9 100644 --- a/shared/kubernetes/provider.go +++ b/shared/kubernetes/provider.go @@ -51,6 +51,8 @@ const ( fieldManagerName = "mizu-manager" procfsVolumeName = "proc" procfsMountPath = "/hostproc" + sysfsVolumeName = "sys" + sysfsMountPath = "/sys" ) func NewProvider(kubeConfigPath string) (*Provider, error) { @@ -795,7 +797,7 @@ func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string, return nil } -func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodMap map[string][]core.Pod, serviceAccountName string, resources shared.Resources, imagePullPolicy core.PullPolicy, mizuApiFilteringOptions api.TrafficFilteringOptions, logLevel logging.Level, serviceMesh bool) error { +func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodMap map[string][]core.Pod, serviceAccountName string, resources shared.Resources, imagePullPolicy core.PullPolicy, mizuApiFilteringOptions api.TrafficFilteringOptions, logLevel logging.Level, serviceMesh bool, tls bool) error { logger.Log.Debugf("Applying %d tapper daemon sets, ns: %s, daemonSetName: %s, podImage: %s, tapperPodName: %s", len(nodeToTappedPodMap), namespace, daemonSetName, podImage, tapperPodName) if len(nodeToTappedPodMap) == 0 { @@ -821,7 +823,15 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac } if serviceMesh { - mizuCmd = append(mizuCmd, "--procfs", procfsMountPath, "--servicemesh") + mizuCmd = append(mizuCmd, "--servicemesh") + } + + if tls { + mizuCmd = append(mizuCmd, "--tls") + } + + if serviceMesh || tls { + mizuCmd = append(mizuCmd, "--procfs", procfsMountPath) } agentContainer := applyconfcore.Container() @@ -829,12 +839,21 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac agentContainer.WithImage(podImage) agentContainer.WithImagePullPolicy(imagePullPolicy) - caps := applyconfcore.Capabilities().WithDrop("ALL").WithAdd("NET_RAW").WithAdd("NET_ADMIN") + caps := applyconfcore.Capabilities().WithDrop("ALL") - if serviceMesh { - caps = caps.WithAdd("SYS_ADMIN") // for reading /proc/PID/net/ns - caps = caps.WithAdd("SYS_PTRACE") // for setting netns to other process - caps = caps.WithAdd("DAC_OVERRIDE") // for reading /proc/PID/environ + caps = caps.WithAdd("NET_RAW").WithAdd("NET_ADMIN") // to listen to traffic using libpcap + + if serviceMesh || tls { + caps = caps.WithAdd("SYS_ADMIN") // to read /proc/PID/net/ns + to install eBPF programs (kernel < 5.8) + caps = caps.WithAdd("SYS_PTRACE") // to set netns to other process + to open libssl.so of other process + + if serviceMesh { + caps = caps.WithAdd("DAC_OVERRIDE") // to read /proc/PID/environ + } + + if tls { + caps = caps.WithAdd("SYS_RESOURCE") // to change rlimits for eBPF + } } agentContainer.WithSecurityContext(applyconfcore.SecurityContext().WithCapabilities(caps)) @@ -910,8 +929,15 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac // procfsVolume := applyconfcore.Volume() procfsVolume.WithName(procfsVolumeName).WithHostPath(applyconfcore.HostPathVolumeSource().WithPath("/proc")) - volumeMount := applyconfcore.VolumeMount().WithName(procfsVolumeName).WithMountPath(procfsMountPath).WithReadOnly(true) - agentContainer.WithVolumeMounts(volumeMount) + procfsVolumeMount := applyconfcore.VolumeMount().WithName(procfsVolumeName).WithMountPath(procfsMountPath).WithReadOnly(true) + agentContainer.WithVolumeMounts(procfsVolumeMount) + + // We need access to /sys in order to install certain eBPF tracepoints + // + sysfsVolume := applyconfcore.Volume() + sysfsVolume.WithName(sysfsVolumeName).WithHostPath(applyconfcore.HostPathVolumeSource().WithPath("/sys")) + sysfsVolumeMount := applyconfcore.VolumeMount().WithName(sysfsVolumeName).WithMountPath(sysfsMountPath).WithReadOnly(true) + agentContainer.WithVolumeMounts(sysfsVolumeMount) volumeName := ConfigMapName configMapVolume := applyconfcore.VolumeApplyConfiguration{ @@ -941,7 +967,7 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac podSpec.WithContainers(agentContainer) podSpec.WithAffinity(affinity) podSpec.WithTolerations(noExecuteToleration, noScheduleToleration) - podSpec.WithVolumes(&configMapVolume, procfsVolume) + podSpec.WithVolumes(&configMapVolume, procfsVolume, sysfsVolume) podTemplate := applyconfcore.PodTemplateSpec() podTemplate.WithLabels(map[string]string{ diff --git a/shared/kubernetes/utils.go b/shared/kubernetes/utils.go index 55a34f97e..e36d83129 100644 --- a/shared/kubernetes/utils.go +++ b/shared/kubernetes/utils.go @@ -30,11 +30,24 @@ func getMinimizedPod(fullPod core.Pod) core.Pod { Name: fullPod.Name, }, Status: v1.PodStatus{ - PodIP: fullPod.Status.PodIP, + PodIP: fullPod.Status.PodIP, + ContainerStatuses: getMinimizedContainerStatuses(fullPod), }, } } +func getMinimizedContainerStatuses(fullPod core.Pod) []v1.ContainerStatus { + result := make([]v1.ContainerStatus, len(fullPod.Status.ContainerStatuses)) + + for i, container := range fullPod.Status.ContainerStatuses { + result[i] = v1.ContainerStatus{ + ContainerID: container.ContainerID, + } + } + + return result +} + func excludeMizuPods(pods []core.Pod) []core.Pod { mizuPrefixRegex := regexp.MustCompile("^" + MizuResourcesPrefix) diff --git a/tap/go.mod b/tap/go.mod index 108fc17ba..2429f2f87 100644 --- a/tap/go.mod +++ b/tap/go.mod @@ -4,6 +4,8 @@ go 1.17 require ( github.com/bradleyfalzon/tlsx v0.0.0-20170624122154-28fd0e59bac4 + github.com/cilium/ebpf v0.8.0 + github.com/go-errors/errors v1.4.2 github.com/google/gopacket v1.1.19 github.com/up9inc/mizu/shared v0.0.0 github.com/up9inc/mizu/tap/api v0.0.0 diff --git a/tap/go.sum b/tap/go.sum index fd2265d2e..bed20da3d 100644 --- a/tap/go.sum +++ b/tap/go.sum @@ -102,6 +102,8 @@ github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cilium/ebpf v0.8.0 h1:2V6KSg3FRADVU2BMIRemZ0hV+9OM+aAHhZDjQyjJTAs= +github.com/cilium/ebpf v0.8.0/go.mod h1:f5zLIM0FSNuAkSyLAN7X+Hy6yznlF1mNiWUMfxMtrgk= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -161,6 +163,8 @@ github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYF github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss= +github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= @@ -169,6 +173,7 @@ github.com/fvbommel/sortorder v1.0.2/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui72 github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -366,6 +371,8 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -481,6 +488,8 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -788,6 +797,7 @@ golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/tap/passive_tapper.go b/tap/passive_tapper.go index 2779295e9..9a5202685 100644 --- a/tap/passive_tapper.go +++ b/tap/passive_tapper.go @@ -21,6 +21,7 @@ import ( "github.com/up9inc/mizu/tap/api" "github.com/up9inc/mizu/tap/diagnose" "github.com/up9inc/mizu/tap/source" + "github.com/up9inc/mizu/tap/tlstapper" v1 "k8s.io/api/core/v1" ) @@ -53,6 +54,7 @@ var promisc = flag.Bool("promisc", true, "Set promiscuous mode") var staleTimeoutSeconds = flag.Int("staletimout", 120, "Max time in seconds to keep connections which don't transmit data") var pids = flag.String("pids", "", "A comma separated list of PIDs to capture their network namespaces") var servicemesh = flag.Bool("servicemesh", false, "Record decrypted traffic if the cluster is configured with a service mesh and with mtls") +var tls = flag.Bool("tls", false, "Enable TLS tapper") var memprofile = flag.String("memprofile", "", "Write memory profile") @@ -95,6 +97,15 @@ func StartPassiveTapper(opts *TapOpts, outputItems chan *api.OutputChannelItem, tapTargets = opts.FilterAuthorities } + if *tls { + for _, e := range extensions { + if e.Protocol.Name == "http" { + startTlsTapper(e, outputItems, options) + break + } + } + } + if GetMemoryProfilingEnabled() { diagnose.StartMemoryProfiler(os.Getenv(MemoryProfilingDumpPath), os.Getenv(MemoryProfilingTimeIntervalSeconds)) } @@ -232,3 +243,35 @@ func startPassiveTapper(opts *TapOpts, outputItems chan *api.OutputChannelItem) diagnose.TapErrors.PrintSummary() logger.Log.Infof("AppStats: %v", diagnose.AppStats) } + +func startTlsTapper(extension *api.Extension, outputItems chan *api.OutputChannelItem, options *api.TrafficFilteringOptions) { + tls := tlstapper.TlsTapper{} + tlsPerfBufferSize := os.Getpagesize() * 100 + + if err := tls.Init(tlsPerfBufferSize); err != nil { + tlstapper.LogError(err) + return + } + + // A quick way to instrument libssl.so without PID filtering - used for debuging and troubleshooting + // + if os.Getenv("MIZU_GLOBAL_SSL_LIBRARY") != "" { + if err := tls.GlobalTap(os.Getenv("MIZU_GLOBAL_SSL_LIBRARY")); err != nil { + tlstapper.LogError(err) + return + } + } + + if err := tlstapper.UpdateTapTargets(&tls, &tapTargets, *procfs); err != nil { + tlstapper.LogError(err) + return + } + + var emitter api.Emitter = &api.Emitting{ + AppStats: &diagnose.AppStats, + OutputChannel: outputItems, + } + + poller := tlstapper.NewTlsPoller(&tls, extension) + go poller.Poll(extension, emitter, options) +} diff --git a/tap/tlstapper/bpf-builder/Dockerfile b/tap/tlstapper/bpf-builder/Dockerfile new file mode 100644 index 000000000..49b5dce66 --- /dev/null +++ b/tap/tlstapper/bpf-builder/Dockerfile @@ -0,0 +1,5 @@ +FROM alpine:3 + +RUN apk --no-cache update && apk --no-cache add clang llvm libbpf-dev go linux-headers + +WORKDIR /mizu diff --git a/tap/tlstapper/bpf-builder/README.md b/tap/tlstapper/bpf-builder/README.md new file mode 100644 index 000000000..fd60ecec7 --- /dev/null +++ b/tap/tlstapper/bpf-builder/README.md @@ -0,0 +1,16 @@ + +# Bpf builder + +Currently we push the ebpf `*.o` files to source control, the motivation for it is to avoid the need for everyone to compile it in their PC. + +This directory helps those who do want to build the .o files, it also serve as a documentation for the process of compiling the ebpf code. + +## How to run ebpf-builder + +From you shell, go to this directory and run `./build.sh` + +Once the docker finished successfully, make sure to commit the four relevant files. +> tlstapper_bpfeb.go +> tlstapper_bpfel.go +> tlstapper_bpfeb.o +> tlstapper_bpfel.o diff --git a/tap/tlstapper/bpf-builder/build.sh b/tap/tlstapper/bpf-builder/build.sh new file mode 100755 index 000000000..7de745d96 --- /dev/null +++ b/tap/tlstapper/bpf-builder/build.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +MIZU_HOME=$(realpath ../../../) + +docker build -t mizu-ebpf-builder . || exit 1 + +docker run --rm \ + --name mizu-ebpf-builder \ + -v $MIZU_HOME:/mizu \ + -it mizu-ebpf-builder \ + sh -c " + go generate tap/tlstapper/tls_tapper.go + chown $(id -u):$(id -g) tap/tlstapper/tlstapper_bpfeb.go + chown $(id -u):$(id -g) tap/tlstapper/tlstapper_bpfeb.o + chown $(id -u):$(id -g) tap/tlstapper/tlstapper_bpfel.go + chown $(id -u):$(id -g) tap/tlstapper/tlstapper_bpfel.o + " || exit 1 diff --git a/tap/tlstapper/bpf/fd_to_address_tracepoints.c b/tap/tlstapper/bpf/fd_to_address_tracepoints.c new file mode 100644 index 000000000..cc9ee81c6 --- /dev/null +++ b/tap/tlstapper/bpf/fd_to_address_tracepoints.c @@ -0,0 +1,215 @@ +/* +Note: This file is licenced differently from the rest of the project +SPDX-License-Identifier: GPL-2.0 +Copyright (C) UP9 Inc. +*/ + +#include "include/headers.h" +#include "include/util.h" +#include "include/maps.h" +#include "include/pids.h" + +struct accept_info { + __u64* sockaddr; + __u32* addrlen; +}; + +BPF_HASH(accept_syscall_context, __u64, struct accept_info); + +struct sys_enter_accept4_ctx { + __u64 __unused_syscall_header; + __u32 __unused_syscall_nr; + + __u64 fd; + __u64* sockaddr; + __u32* addrlen; +}; + +SEC("tracepoint/syscalls/sys_enter_accept4") +void sys_enter_accept4(struct sys_enter_accept4_ctx *ctx) { + __u64 id = bpf_get_current_pid_tgid(); + + if (!should_tap(id >> 32)) { + return; + } + + struct accept_info info = {}; + + info.sockaddr = ctx->sockaddr; + info.addrlen = ctx->addrlen; + + long err = bpf_map_update_elem(&accept_syscall_context, &id, &info, BPF_ANY); + + if (err != 0) { + char msg[] = "Error putting accept info (id: %ld) (err: %ld)"; + bpf_trace_printk(msg, sizeof(msg), id, err); + return; + } +} + +struct sys_exit_accept4_ctx { + __u64 __unused_syscall_header; + __u32 __unused_syscall_nr; + + __u64 ret; +}; + +SEC("tracepoint/syscalls/sys_exit_accept4") +void sys_exit_accept4(struct sys_exit_accept4_ctx *ctx) { + __u64 id = bpf_get_current_pid_tgid(); + + if (!should_tap(id >> 32)) { + return; + } + + if (ctx->ret < 0) { + bpf_map_delete_elem(&accept_syscall_context, &id); + return; + } + + struct accept_info *infoPtr = bpf_map_lookup_elem(&accept_syscall_context, &id); + + if (infoPtr == 0) { + return; + } + + struct accept_info info; + long err = bpf_probe_read(&info, sizeof(struct accept_info), infoPtr); + + bpf_map_delete_elem(&accept_syscall_context, &id); + + if (err != 0) { + char msg[] = "Error reading accept info from accept syscall (id: %ld) (err: %ld)"; + bpf_trace_printk(msg, sizeof(msg), id, err); + return; + } + + __u32 addrlen; + bpf_probe_read(&addrlen, sizeof(__u32), info.addrlen); + + if (addrlen != 16) { + // Currently only ipv4 is supported linux-src/include/linux/inet.h + return; + } + + struct fd_info fdinfo = { + .flags = 0 + }; + + bpf_probe_read(fdinfo.ipv4_addr, sizeof(fdinfo.ipv4_addr), info.sockaddr); + + __u32 pid = id >> 32; + __u32 fd = (__u32) ctx->ret; + + __u64 key = (__u64) pid << 32 | fd; + err = bpf_map_update_elem(&file_descriptor_to_ipv4, &key, &fdinfo, BPF_ANY); + + if (err != 0) { + char msg[] = "Error putting fd to address mapping from accept (key: %ld) (err: %ld)"; + bpf_trace_printk(msg, sizeof(msg), key, err); + return; + } +} + +struct connect_info { + __u64 fd; + __u64* sockaddr; + __u32 addrlen; +}; + +BPF_HASH(connect_syscall_info, __u64, struct connect_info); + +struct sys_enter_connect_ctx { + __u64 __unused_syscall_header; + __u32 __unused_syscall_nr; + + __u64 fd; + __u64* sockaddr; + __u32 addrlen; +}; + +SEC("tracepoint/syscalls/sys_enter_connect") +void sys_enter_connect(struct sys_enter_connect_ctx *ctx) { + __u64 id = bpf_get_current_pid_tgid(); + + if (!should_tap(id >> 32)) { + return; + } + + struct connect_info info = {}; + + info.sockaddr = ctx->sockaddr; + info.addrlen = ctx->addrlen; + info.fd = ctx->fd; + + long err = bpf_map_update_elem(&connect_syscall_info, &id, &info, BPF_ANY); + + if (err != 0) { + char msg[] = "Error putting connect info (id: %ld) (err: %ld)"; + bpf_trace_printk(msg, sizeof(msg), id, err); + return; + } +} + +struct sys_exit_connect_ctx { + __u64 __unused_syscall_header; + __u32 __unused_syscall_nr; + + __u64 ret; +}; + +SEC("tracepoint/syscalls/sys_exit_connect") +void sys_exit_connect(struct sys_exit_connect_ctx *ctx) { + __u64 id = bpf_get_current_pid_tgid(); + + if (!should_tap(id >> 32)) { + return; + } + + // Commented because of async connect which set errno to EINPROGRESS + // + // if (ctx->ret != 0) { + // bpf_map_delete_elem(&accept_syscall_context, &id); + // return; + // } + + struct connect_info *infoPtr = bpf_map_lookup_elem(&connect_syscall_info, &id); + + if (infoPtr == 0) { + return; + } + + struct connect_info info; + long err = bpf_probe_read(&info, sizeof(struct connect_info), infoPtr); + + bpf_map_delete_elem(&connect_syscall_info, &id); + + if (err != 0) { + char msg[] = "Error reading connect info from connect syscall (id: %ld) (err: %ld)"; + bpf_trace_printk(msg, sizeof(msg), id, err); + return; + } + + if (info.addrlen != 16) { + // Currently only ipv4 is supported linux-src/include/linux/inet.h + return; + } + + struct fd_info fdinfo = { + .flags = FLAGS_IS_CLIENT_BIT + }; + + bpf_probe_read(fdinfo.ipv4_addr, sizeof(fdinfo.ipv4_addr), info.sockaddr); + + __u32 pid = id >> 32; + __u32 fd = (__u32) info.fd; + + __u64 key = (__u64) pid << 32 | fd; + err = bpf_map_update_elem(&file_descriptor_to_ipv4, &key, &fdinfo, BPF_ANY); + + if (err != 0) { + char msg[] = "Error putting fd to address mapping from connect (key: %ld) (err: %ld)"; + bpf_trace_printk(msg, sizeof(msg), key, err); + return; + } +} diff --git a/tap/tlstapper/bpf/fd_tracepoints.c b/tap/tlstapper/bpf/fd_tracepoints.c new file mode 100644 index 000000000..7a18948c9 --- /dev/null +++ b/tap/tlstapper/bpf/fd_tracepoints.c @@ -0,0 +1,96 @@ +/* +Note: This file is licenced differently from the rest of the project +SPDX-License-Identifier: GPL-2.0 +Copyright (C) UP9 Inc. +*/ + +#include "include/headers.h" +#include "include/util.h" +#include "include/maps.h" +#include "include/pids.h" + +struct sys_enter_read_ctx { + __u64 __unused_syscall_header; + __u32 __unused_syscall_nr; + + __u64 fd; + __u64* buf; + __u64 count; +}; + +SEC("tracepoint/syscalls/sys_enter_read") +void sys_enter_read(struct sys_enter_read_ctx *ctx) { + __u64 id = bpf_get_current_pid_tgid(); + + if (!should_tap(id >> 32)) { + return; + } + + struct ssl_info *infoPtr = bpf_map_lookup_elem(&ssl_read_context, &id); + + if (infoPtr == 0) { + return; + } + + struct ssl_info info; + long err = bpf_probe_read(&info, sizeof(struct ssl_info), infoPtr); + + if (err != 0) { + char msg[] = "Error reading read info from read syscall (id: %ld) (err: %ld)"; + bpf_trace_printk(msg, sizeof(msg), id, err); + return; + } + + info.fd = ctx->fd; + + err = bpf_map_update_elem(&ssl_read_context, &id, &info, BPF_ANY); + + if (err != 0) { + char msg[] = "Error putting file descriptor from read syscall (id: %ld) (err: %ld)"; + bpf_trace_printk(msg, sizeof(msg), id, err); + return; + } +} + +struct sys_enter_write_ctx { + __u64 __unused_syscall_header; + __u32 __unused_syscall_nr; + + __u64 fd; + __u64* buf; + __u64 count; +}; + +SEC("tracepoint/syscalls/sys_enter_write") +void sys_enter_write(struct sys_enter_write_ctx *ctx) { + __u64 id = bpf_get_current_pid_tgid(); + + if (!should_tap(id >> 32)) { + return; + } + + struct ssl_info *infoPtr = bpf_map_lookup_elem(&ssl_write_context, &id); + + if (infoPtr == 0) { + return; + } + + struct ssl_info info; + long err = bpf_probe_read(&info, sizeof(struct ssl_info), infoPtr); + + if (err != 0) { + char msg[] = "Error reading write context from write syscall (id: %ld) (err: %ld)"; + bpf_trace_printk(msg, sizeof(msg), id, err); + return; + } + + info.fd = ctx->fd; + + err = bpf_map_update_elem(&ssl_write_context, &id, &info, BPF_ANY); + + if (err != 0) { + char msg[] = "Error putting file descriptor from write syscall (id: %ld) (err: %ld)"; + bpf_trace_printk(msg, sizeof(msg), id, err); + return; + } +} diff --git a/tap/tlstapper/bpf/include/headers.h b/tap/tlstapper/bpf/include/headers.h new file mode 100644 index 000000000..8078051af --- /dev/null +++ b/tap/tlstapper/bpf/include/headers.h @@ -0,0 +1,16 @@ +/* +Note: This file is licenced differently from the rest of the project +SPDX-License-Identifier: GPL-2.0 +Copyright (C) UP9 Inc. +*/ + +#ifndef __HEADERS__ +#define __HEADERS__ + +#include +#include +#include +#include +#include "bpf/bpf_tracing.h" + +#endif /* __HEADERS__ */ diff --git a/tap/tlstapper/bpf/include/maps.h b/tap/tlstapper/bpf/include/maps.h new file mode 100644 index 000000000..5d162ec09 --- /dev/null +++ b/tap/tlstapper/bpf/include/maps.h @@ -0,0 +1,63 @@ +/* +Note: This file is licenced differently from the rest of the project +SPDX-License-Identifier: GPL-2.0 +Copyright (C) UP9 Inc. +*/ + +#ifndef __MAPS__ +#define __MAPS__ + +#define FLAGS_IS_CLIENT_BIT (1 << 0) +#define FLAGS_IS_READ_BIT (1 << 1) + +// The same struct can be found in Chunk.go +// +// Be careful when editing, alignment and padding should be exactly the same in go/c. +// +struct tlsChunk { + __u32 pid; + __u32 tgid; + __u32 len; + __u32 recorded; + __u32 fd; + __u32 flags; + __u8 address[16]; + __u8 data[4096]; // Must be N^2 +}; + +struct ssl_info { + void* buffer; + __u32 fd; + + // for ssl_write and ssl_read must be zero + // for ssl_write_ex and ssl_read_ex save the *written/*readbytes pointer. + // + size_t *count_ptr; +}; + +struct fd_info { + __u8 ipv4_addr[16]; // struct sockaddr (linux-src/include/linux/socket.h) + __u8 flags; +}; + +#define BPF_MAP(_name, _type, _key_type, _value_type, _max_entries) \ + struct bpf_map_def SEC("maps") _name = { \ + .type = _type, \ + .key_size = sizeof(_key_type), \ + .value_size = sizeof(_value_type), \ + .max_entries = _max_entries, \ + }; + +#define BPF_HASH(_name, _key_type, _value_type) \ + BPF_MAP(_name, BPF_MAP_TYPE_HASH, _key_type, _value_type, 4096) + +#define BPF_PERF_OUTPUT(_name) \ + BPF_MAP(_name, BPF_MAP_TYPE_PERF_EVENT_ARRAY, int, __u32, 1024) + +BPF_HASH(pids_map, __u32, __u32); +BPF_HASH(ssl_write_context, __u64, struct ssl_info); +BPF_HASH(ssl_read_context, __u64, struct ssl_info); +BPF_HASH(file_descriptor_to_ipv4, __u64, struct fd_info); +BPF_PERF_OUTPUT(chunks_buffer); + +#endif /* __MAPS__ */ diff --git a/tap/tlstapper/bpf/include/pids.h b/tap/tlstapper/bpf/include/pids.h new file mode 100644 index 000000000..ecc0c2427 --- /dev/null +++ b/tap/tlstapper/bpf/include/pids.h @@ -0,0 +1,23 @@ +/* +Note: This file is licenced differently from the rest of the project +SPDX-License-Identifier: GPL-2.0 +Copyright (C) UP9 Inc. +*/ + +#ifndef __PIDS__ +#define __PIDS__ + +int should_tap(__u32 pid) { + __u32* shouldTap = bpf_map_lookup_elem(&pids_map, &pid); + + if (shouldTap != NULL && *shouldTap == 1) { + return 1; + } + + __u32 globalPid = 0; + __u32* shouldTapGlobally = bpf_map_lookup_elem(&pids_map, &globalPid); + + return shouldTapGlobally != NULL && *shouldTapGlobally == 1; +} + +#endif /* __PIDS__ */ diff --git a/tap/tlstapper/bpf/include/util.h b/tap/tlstapper/bpf/include/util.h new file mode 100644 index 000000000..92e5d7b41 --- /dev/null +++ b/tap/tlstapper/bpf/include/util.h @@ -0,0 +1,12 @@ +/* +Note: This file is licenced differently from the rest of the project +SPDX-License-Identifier: GPL-2.0 +Copyright (C) UP9 Inc. +*/ + +#ifndef __UTIL__ +#define __UTIL__ + +#define MIN(a,b) (((a)<(b))?(a):(b)) + +#endif /* __UTIL__ */ diff --git a/tap/tlstapper/bpf/openssl_uprobes.c b/tap/tlstapper/bpf/openssl_uprobes.c new file mode 100644 index 000000000..c6bbf27c1 --- /dev/null +++ b/tap/tlstapper/bpf/openssl_uprobes.c @@ -0,0 +1,198 @@ +/* +Note: This file is licenced differently from the rest of the project +SPDX-License-Identifier: GPL-2.0 +Copyright (C) UP9 Inc. +*/ + +#include "include/headers.h" +#include "include/util.h" +#include "include/maps.h" +#include "include/pids.h" + +// Heap-like area for eBPF programs - stack size limited to 512 bytes, we must use maps for bigger (chunk) objects. +// +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, 1); + __type(key, int); + __type(value, struct tlsChunk); +} heap SEC(".maps"); + +static __always_inline int ssl_uprobe(void* ssl, void* buffer, int num, struct bpf_map_def* map_fd, size_t *count_ptr) { + __u64 id = bpf_get_current_pid_tgid(); + + if (!should_tap(id >> 32)) { + return 0; + } + + struct ssl_info info = {}; + + info.fd = -1; + info.count_ptr = count_ptr; + info.buffer = buffer; + + long err = bpf_map_update_elem(map_fd, &id, &info, BPF_ANY); + + if (err != 0) { + char msg[] = "Error putting ssl context (id: %ld) (err: %ld)"; + bpf_trace_printk(msg, sizeof(msg), id, err); + return 0; + } + + return 0; +} + +static __always_inline int ssl_uretprobe(struct pt_regs *ctx, struct bpf_map_def* map_fd, __u32 flags) { + __u64 id = bpf_get_current_pid_tgid(); + + if (!should_tap(id >> 32)) { + return 0; + } + + struct ssl_info *infoPtr = bpf_map_lookup_elem(map_fd, &id); + + if (infoPtr == 0) { + char msg[] = "Error getting ssl context info (id: %ld)"; + bpf_trace_printk(msg, sizeof(msg), id); + return 0; + } + + struct ssl_info info; + long err = bpf_probe_read(&info, sizeof(struct ssl_info), infoPtr); + + bpf_map_delete_elem(map_fd, &id); + + if (err != 0) { + char msg[] = "Error reading ssl context (id: %ld) (err: %ld)"; + bpf_trace_printk(msg, sizeof(msg), id, err); + return 0; + } + + if (info.fd == -1) { + char msg[] = "File descriptor is missing from ssl info (id: %ld)"; + bpf_trace_printk(msg, sizeof(msg), id); + return 0; + } + + int countBytes = PT_REGS_RC(ctx); + + if (info.count_ptr != 0) { + // ssl_read_ex and ssl_write_ex return 1 for success + // + if (countBytes != 1) { + return 0; + } + + size_t tempCount; + long err = bpf_probe_read(&tempCount, sizeof(size_t), (void*) info.count_ptr); + + if (err != 0) { + char msg[] = "Error reading bytes count of _ex (id: %ld) (err: %ld)"; + bpf_trace_printk(msg, sizeof(msg), id, err); + return 0; + } + + countBytes = tempCount; + } + + if (countBytes <= 0) { + return 0; + } + + struct tlsChunk* c; + int zero = 0; + + // If other thread, running on the same CPU get to this point at the same time like us + // the data will be corrupted - protection may be added in the future + // + c = bpf_map_lookup_elem(&heap, &zero); + + if (!c) { + char msg[] = "Unable to allocate chunk (id: %ld)"; + bpf_trace_printk(msg, sizeof(msg), id); + return 0; + } + + size_t recorded = MIN(countBytes, sizeof(c->data)); + + c->flags = flags; + c->pid = id >> 32; + c->tgid = id; + c->len = countBytes; + c->recorded = recorded; + c->fd = info.fd; + + // This ugly trick is for the ebpf verifier happiness + // + if (recorded == sizeof(c->data)) { + err = bpf_probe_read(c->data, sizeof(c->data), info.buffer); + } else { + recorded &= sizeof(c->data) - 1; // Buffer must be N^2 + err = bpf_probe_read(c->data, recorded, info.buffer); + } + + if (err != 0) { + char msg[] = "Error reading from ssl buffer %ld - %ld"; + bpf_trace_printk(msg, sizeof(msg), id, err); + return 0; + } + + __u32 pid = id >> 32; + __u32 fd = info.fd; + __u64 key = (__u64) pid << 32 | fd; + + struct fd_info *fdinfo = bpf_map_lookup_elem(&file_descriptor_to_ipv4, &key); + + if (fdinfo != 0) { + err = bpf_probe_read(c->address, sizeof(c->address), fdinfo->ipv4_addr); + c->flags |= (fdinfo->flags & FLAGS_IS_CLIENT_BIT); + + if (err != 0) { + char msg[] = "Error reading from fd address %ld - %ld"; + bpf_trace_printk(msg, sizeof(msg), id, err); + } + } + + bpf_perf_event_output(ctx, &chunks_buffer, BPF_F_CURRENT_CPU, c, sizeof(struct tlsChunk)); + return 0; +} + +SEC("uprobe/ssl_write") +int BPF_KPROBE(ssl_write, void* ssl, void* buffer, int num) { + return ssl_uprobe(ssl, buffer, num, &ssl_write_context, 0); +} + +SEC("uretprobe/ssl_write") +int BPF_KPROBE(ssl_ret_write) { + return ssl_uretprobe(ctx, &ssl_write_context, 0); +} + +SEC("uprobe/ssl_read") +int BPF_KPROBE(ssl_read, void* ssl, void* buffer, int num) { + return ssl_uprobe(ssl, buffer, num, &ssl_read_context, 0); +} + +SEC("uretprobe/ssl_read") +int BPF_KPROBE(ssl_ret_read) { + return ssl_uretprobe(ctx, &ssl_read_context, FLAGS_IS_READ_BIT); +} + +SEC("uprobe/ssl_write_ex") +int BPF_KPROBE(ssl_write_ex, void* ssl, void* buffer, size_t num, size_t *written) { + return ssl_uprobe(ssl, buffer, num, &ssl_write_context, written); +} + +SEC("uretprobe/ssl_write_ex") +int BPF_KPROBE(ssl_ret_write_ex) { + return ssl_uretprobe(ctx, &ssl_write_context, 0); +} + +SEC("uprobe/ssl_read_ex") +int BPF_KPROBE(ssl_read_ex, void* ssl, void* buffer, size_t num, size_t *readbytes) { + return ssl_uprobe(ssl, buffer, num, &ssl_read_context, readbytes); +} + +SEC("uretprobe/ssl_read_ex") +int BPF_KPROBE(ssl_ret_read_ex) { + return ssl_uretprobe(ctx, &ssl_read_context, FLAGS_IS_READ_BIT); +} diff --git a/tap/tlstapper/bpf/tls_tapper.c b/tap/tlstapper/bpf/tls_tapper.c new file mode 100644 index 000000000..f48fec6fb --- /dev/null +++ b/tap/tlstapper/bpf/tls_tapper.c @@ -0,0 +1,18 @@ +/* +Note: This file is licenced differently from the rest of the project +SPDX-License-Identifier: GPL-2.0 +Copyright (C) UP9 Inc. +*/ + +#include "include/headers.h" +#include "include/util.h" +#include "include/maps.h" +#include "include/pids.h" + +// To avoid multiple .o files +// +#include "openssl_uprobes.c" +#include "fd_tracepoints.c" +#include "fd_to_address_tracepoints.c" + +char _license[] SEC("license") = "GPL"; diff --git a/tap/tlstapper/chunk.go b/tap/tlstapper/chunk.go new file mode 100644 index 000000000..48fcf17e9 --- /dev/null +++ b/tap/tlstapper/chunk.go @@ -0,0 +1,70 @@ +package tlstapper + +import ( + "bytes" + "encoding/binary" + "net" + + "github.com/go-errors/errors" +) + +const FLAGS_IS_CLIENT_BIT int32 = (1 << 0) +const FLAGS_IS_READ_BIT int32 = (1 << 1) + +// The same struct can be found in maps.h +// +// Be careful when editing, alignment and padding should be exactly the same in go/c. +// +type tlsChunk struct { + Pid int32 + Tgid int32 + Len int32 + Recorded int32 + Fd int32 + Flags int32 + Address [16]byte + Data [4096]byte +} + +func (c *tlsChunk) getAddress() (net.IP, uint16, error) { + address := bytes.NewReader(c.Address[:]) + var family uint16 + var port uint16 + var ip32 uint32 + + if err := binary.Read(address, binary.BigEndian, &family); err != nil { + return nil, 0, errors.Wrap(err, 0) + } + + if err := binary.Read(address, binary.BigEndian, &port); err != nil { + return nil, 0, errors.Wrap(err, 0) + } + + if err := binary.Read(address, binary.BigEndian, &ip32); err != nil { + return nil, 0, errors.Wrap(err, 0) + } + + ip := net.IP{uint8(ip32 >> 24), uint8(ip32 >> 16), uint8(ip32 >> 8), uint8(ip32)} + + return ip, port, nil +} + +func (c *tlsChunk) isClient() bool { + return c.Flags&FLAGS_IS_CLIENT_BIT != 0 +} + +func (c *tlsChunk) isServer() bool { + return !c.isClient() +} + +func (c *tlsChunk) isRead() bool { + return c.Flags&FLAGS_IS_READ_BIT != 0 +} + +func (c *tlsChunk) isWrite() bool { + return !c.isRead() +} + +func (c *tlsChunk) getRecordedData() []byte { + return c.Data[:c.Recorded] +} diff --git a/tap/tlstapper/ssllib_finder.go b/tap/tlstapper/ssllib_finder.go new file mode 100644 index 000000000..bdeccd961 --- /dev/null +++ b/tap/tlstapper/ssllib_finder.go @@ -0,0 +1,63 @@ +package tlstapper + +import ( + "bufio" + "fmt" + "os" + "strings" + + "github.com/go-errors/errors" + "github.com/up9inc/mizu/shared/logger" +) + +func findSsllib(procfs string, pid uint32) (string, error) { + binary, err := os.Readlink(fmt.Sprintf("%s/%d/exe", procfs, pid)) + + if err != nil { + return "", errors.Wrap(err, 0) + } + + logger.Log.Debugf("Binary file for %v = %v", pid, binary) + + if strings.HasSuffix(binary, "/node") { + return findLibraryByPid(procfs, pid, binary) + } else { + return findLibraryByPid(procfs, pid, "libssl.so") + } +} + +func findLibraryByPid(procfs string, pid uint32, libraryName string) (string, error) { + file, err := os.Open(fmt.Sprintf("%v/%v/maps", procfs, pid)) + + if err != nil { + return "", err + } + + defer file.Close() + scanner := bufio.NewScanner(file) + scanner.Split(bufio.ScanLines) + + for scanner.Scan() { + parts := strings.Fields(scanner.Text()) + + if len(parts) <= 5 { + continue + } + + filepath := parts[5] + + if !strings.Contains(filepath, libraryName) { + continue + } + + fullpath := fmt.Sprintf("%v/%v/root/%v", procfs, pid, filepath) + + if _, err := os.Stat(fullpath); os.IsNotExist(err) { + continue + } + + return fullpath, nil + } + + return "", errors.Errorf("%s not found for PID %d", libraryName, pid) +} diff --git a/tap/tlstapper/ssllib_hooks.go b/tap/tlstapper/ssllib_hooks.go new file mode 100644 index 000000000..266dd9f33 --- /dev/null +++ b/tap/tlstapper/ssllib_hooks.go @@ -0,0 +1,153 @@ +package tlstapper + +import ( + "github.com/cilium/ebpf/link" + "github.com/go-errors/errors" +) + +type sslHooks struct { + sslWriteProbe link.Link + sslWriteRetProbe link.Link + sslReadProbe link.Link + sslReadRetProbe link.Link + sslWriteExProbe link.Link + sslWriteExRetProbe link.Link + sslReadExProbe link.Link + sslReadExRetProbe link.Link +} + +func (s *sslHooks) installUprobes(bpfObjects *tlsTapperObjects, sslLibraryPath string) error { + sslLibrary, err := link.OpenExecutable(sslLibraryPath) + + if err != nil { + return errors.Wrap(err, 0) + } + + sslOffsets, err := getSslOffsets(sslLibraryPath) + + if err != nil { + return errors.Wrap(err, 0) + } + + return s.installSslHooks(bpfObjects, sslLibrary, sslOffsets) +} + +func (s *sslHooks) installSslHooks(bpfObjects *tlsTapperObjects, sslLibrary *link.Executable, offsets sslOffsets) error { + var err error + + s.sslWriteProbe, err = sslLibrary.Uprobe("SSL_write", bpfObjects.SslWrite, &link.UprobeOptions{ + Offset: offsets.SslWriteOffset, + }) + + if err != nil { + return errors.Wrap(err, 0) + } + + s.sslWriteRetProbe, err = sslLibrary.Uretprobe("SSL_write", bpfObjects.SslRetWrite, &link.UprobeOptions{ + Offset: offsets.SslWriteOffset, + }) + + if err != nil { + return errors.Wrap(err, 0) + } + + s.sslReadProbe, err = sslLibrary.Uprobe("SSL_read", bpfObjects.SslRead, &link.UprobeOptions{ + Offset: offsets.SslReadOffset, + }) + + if err != nil { + return errors.Wrap(err, 0) + } + + s.sslReadRetProbe, err = sslLibrary.Uretprobe("SSL_read", bpfObjects.SslRetRead, &link.UprobeOptions{ + Offset: offsets.SslReadOffset, + }) + + if err != nil { + return errors.Wrap(err, 0) + } + + if offsets.SslWriteExOffset != 0 { + s.sslWriteExProbe, err = sslLibrary.Uprobe("SSL_write_ex", bpfObjects.SslWriteEx, &link.UprobeOptions{ + Offset: offsets.SslWriteExOffset, + }) + + if err != nil { + return errors.Wrap(err, 0) + } + + s.sslWriteExRetProbe, err = sslLibrary.Uretprobe("SSL_write_ex", bpfObjects.SslRetWriteEx, &link.UprobeOptions{ + Offset: offsets.SslWriteExOffset, + }) + + if err != nil { + return errors.Wrap(err, 0) + } + } + + if offsets.SslReadExOffset != 0 { + s.sslReadExProbe, err = sslLibrary.Uprobe("SSL_read_ex", bpfObjects.SslReadEx, &link.UprobeOptions{ + Offset: offsets.SslReadExOffset, + }) + + if err != nil { + return errors.Wrap(err, 0) + } + + s.sslReadExRetProbe, err = sslLibrary.Uretprobe("SSL_read_ex", bpfObjects.SslRetReadEx, &link.UprobeOptions{ + Offset: offsets.SslReadExOffset, + }) + + if err != nil { + return errors.Wrap(err, 0) + } + } + + return nil +} + +func (s *sslHooks) close() []error { + errors := make([]error, 0) + + if err := s.sslWriteProbe.Close(); err != nil { + errors = append(errors, err) + } + + if err := s.sslWriteRetProbe.Close(); err != nil { + errors = append(errors, err) + } + + if err := s.sslReadProbe.Close(); err != nil { + errors = append(errors, err) + } + + if err := s.sslReadRetProbe.Close(); err != nil { + errors = append(errors, err) + } + + if s.sslWriteExProbe != nil { + if err := s.sslWriteExProbe.Close(); err != nil { + errors = append(errors, err) + } + } + + if s.sslWriteExRetProbe != nil { + if err := s.sslWriteExRetProbe.Close(); err != nil { + errors = append(errors, err) + } + } + + if s.sslReadExProbe != nil { + if err := s.sslReadExProbe.Close(); err != nil { + errors = append(errors, err) + } + } + + if s.sslReadExRetProbe != nil { + if err := s.sslReadExRetProbe.Close(); err != nil { + errors = append(errors, err) + } + } + + return errors +} diff --git a/tap/tlstapper/ssllib_offsets.go b/tap/tlstapper/ssllib_offsets.go new file mode 100644 index 000000000..84de875ae --- /dev/null +++ b/tap/tlstapper/ssllib_offsets.go @@ -0,0 +1,113 @@ +package tlstapper + +import ( + "debug/elf" + "fmt" + + "github.com/go-errors/errors" + "github.com/up9inc/mizu/shared/logger" +) + +type sslOffsets struct { + SslWriteOffset uint64 + SslReadOffset uint64 + SslWriteExOffset uint64 + SslReadExOffset uint64 +} + +func getSslOffsets(sslLibraryPath string) (sslOffsets, error) { + sslElf, err := elf.Open(sslLibraryPath) + + if err != nil { + return sslOffsets{}, errors.Wrap(err, 0) + } + + defer sslElf.Close() + + base, err := findBaseAddress(sslElf, sslLibraryPath) + + if err != nil { + return sslOffsets{}, errors.Wrap(err, 0) + } + + offsets, err := findSslOffsets(sslElf, base) + + if err != nil { + return sslOffsets{}, errors.Wrap(err, 0) + } + + logger.Log.Debugf("Found TLS offsets (base: 0x%X) (write: 0x%X) (read: 0x%X)", base, offsets.SslWriteOffset, offsets.SslReadOffset) + return offsets, nil +} + +func findBaseAddress(sslElf *elf.File, sslLibraryPath string) (uint64, error) { + for _, prog := range sslElf.Progs { + if prog.Type == elf.PT_LOAD { + return prog.Paddr, nil + } + } + + return 0, errors.New(fmt.Sprintf("Program header not found in %v", sslLibraryPath)) +} + +func findSslOffsets(sslElf *elf.File, base uint64) (sslOffsets, error) { + symbolsMap := make(map[string]elf.Symbol) + + if err := buildSymbolsMap(sslElf.Symbols, symbolsMap); err != nil { + return sslOffsets{}, errors.Wrap(err, 0) + } + + if err := buildSymbolsMap(sslElf.DynamicSymbols, symbolsMap); err != nil { + return sslOffsets{}, errors.Wrap(err, 0) + } + + var sslWriteSymbol, sslReadSymbol, sslWriteExSymbol, sslReadExSymbol elf.Symbol + var ok bool + + if sslWriteSymbol, ok = symbolsMap["SSL_write"]; !ok { + return sslOffsets{}, errors.New("SSL_write symbol not found") + } + + if sslReadSymbol, ok = symbolsMap["SSL_read"]; !ok { + return sslOffsets{}, errors.New("SSL_read symbol not found") + } + + var sslWriteExOffset, sslReadExOffset uint64 + + if sslWriteExSymbol, ok = symbolsMap["SSL_write_ex"]; !ok { + sslWriteExOffset = 0 // libssl.so.1.0 doesn't have the _ex functions + } else { + sslWriteExOffset = sslWriteExSymbol.Value - base + } + + if sslReadExSymbol, ok = symbolsMap["SSL_read_ex"]; !ok { + sslReadExOffset = 0 // libssl.so.1.0 doesn't have the _ex functions + } else { + sslReadExOffset = sslReadExSymbol.Value - base + } + + return sslOffsets{ + SslWriteOffset: sslWriteSymbol.Value - base, + SslReadOffset: sslReadSymbol.Value - base, + SslWriteExOffset: sslWriteExOffset, + SslReadExOffset: sslReadExOffset, + }, nil +} + +func buildSymbolsMap(sectionGetter func() ([]elf.Symbol, error), symbols map[string]elf.Symbol) error { + syms, err := sectionGetter() + + if err != nil && !errors.Is(err, elf.ErrNoSymbols) { + return err + } + + for _, sym := range syms { + if elf.ST_TYPE(sym.Info) != elf.STT_FUNC { + continue + } + + symbols[sym.Name] = sym + } + + return nil +} diff --git a/tap/tlstapper/syscall_hooks.go b/tap/tlstapper/syscall_hooks.go new file mode 100644 index 000000000..0fa621496 --- /dev/null +++ b/tap/tlstapper/syscall_hooks.go @@ -0,0 +1,87 @@ +package tlstapper + +import ( + "github.com/cilium/ebpf/link" + "github.com/go-errors/errors" +) + +type syscallHooks struct { + sysEnterRead link.Link + sysEnterWrite link.Link + sysEnterAccept4 link.Link + sysExitAccept4 link.Link + sysEnterConnect link.Link + sysExitConnect link.Link +} + +func (s *syscallHooks) installSyscallHooks(bpfObjects *tlsTapperObjects) error { + var err error + + s.sysEnterRead, err = link.Tracepoint("syscalls", "sys_enter_read", bpfObjects.SysEnterRead) + + if err != nil { + return errors.Wrap(err, 0) + } + + s.sysEnterWrite, err = link.Tracepoint("syscalls", "sys_enter_write", bpfObjects.SysEnterWrite) + + if err != nil { + return errors.Wrap(err, 0) + } + + s.sysEnterAccept4, err = link.Tracepoint("syscalls", "sys_enter_accept4", bpfObjects.SysEnterAccept4) + + if err != nil { + return errors.Wrap(err, 0) + } + + s.sysExitAccept4, err = link.Tracepoint("syscalls", "sys_exit_accept4", bpfObjects.SysExitAccept4) + + if err != nil { + return errors.Wrap(err, 0) + } + + s.sysEnterConnect, err = link.Tracepoint("syscalls", "sys_enter_connect", bpfObjects.SysEnterConnect) + + if err != nil { + return errors.Wrap(err, 0) + } + + s.sysExitConnect, err = link.Tracepoint("syscalls", "sys_exit_connect", bpfObjects.SysExitConnect) + + if err != nil { + return errors.Wrap(err, 0) + } + + return nil +} + +func (s *syscallHooks) close() []error { + errors := make([]error, 0) + + if err := s.sysEnterRead.Close(); err != nil { + errors = append(errors, err) + } + + if err := s.sysEnterWrite.Close(); err != nil { + errors = append(errors, err) + } + + if err := s.sysEnterAccept4.Close(); err != nil { + errors = append(errors, err) + } + + if err := s.sysExitAccept4.Close(); err != nil { + errors = append(errors, err) + } + + if err := s.sysEnterConnect.Close(); err != nil { + errors = append(errors, err) + } + + if err := s.sysExitConnect.Close(); err != nil { + errors = append(errors, err) + } + + return errors +} diff --git a/tap/tlstapper/tls_poller.go b/tap/tlstapper/tls_poller.go new file mode 100644 index 000000000..2b209b086 --- /dev/null +++ b/tap/tlstapper/tls_poller.go @@ -0,0 +1,162 @@ +package tlstapper + +import ( + "bufio" + "fmt" + "net" + + "encoding/hex" + "os" + "strconv" + "strings" + + "github.com/up9inc/mizu/shared/logger" + "github.com/up9inc/mizu/tap/api" +) + +const UNKNOWN_PORT uint16 = 80 +const UNKNOWN_HOST string = "127.0.0.1" + +type tlsPoller struct { + tls *TlsTapper + readers map[string]*tlsReader + closedReaders chan string + reqResMatcher api.RequestResponseMatcher +} + +func NewTlsPoller(tls *TlsTapper, extension *api.Extension) *tlsPoller { + return &tlsPoller{ + tls: tls, + readers: make(map[string]*tlsReader), + closedReaders: make(chan string, 100), + reqResMatcher: extension.Dissector.NewResponseRequestMatcher(), + } +} + +func (p *tlsPoller) Poll(extension *api.Extension, + emitter api.Emitter, options *api.TrafficFilteringOptions) { + + chunks := make(chan *tlsChunk) + + go p.tls.pollPerf(chunks) + + for { + select { + case chunk, ok := <-chunks: + if !ok { + return + } + + if err := p.handleTlsChunk(chunk, extension, emitter, options); err != nil { + LogError(err) + } + case key := <-p.closedReaders: + delete(p.readers, key) + } + } +} + +func (p *tlsPoller) handleTlsChunk(chunk *tlsChunk, extension *api.Extension, + emitter api.Emitter, options *api.TrafficFilteringOptions) error { + ip, port, err := chunk.getAddress() + + if err != nil { + return err + } + + key := buildTlsKey(chunk, ip, port) + reader, exists := p.readers[key] + + if !exists { + reader = p.startNewTlsReader(chunk, ip, port, key, extension, emitter, options) + p.readers[key] = reader + } + + reader.chunks <- chunk + + if os.Getenv("MIZU_VERBOSE_TLS_TAPPER") == "true" { + logTls(chunk, ip, port) + } + + return nil +} + +func (p *tlsPoller) startNewTlsReader(chunk *tlsChunk, ip net.IP, port uint16, key string, extension *api.Extension, + emitter api.Emitter, options *api.TrafficFilteringOptions) *tlsReader { + + reader := &tlsReader{ + key: key, + chunks: make(chan *tlsChunk, 1), + doneHandler: func(r *tlsReader) { + p.closeReader(key, r) + }, + } + + isRequest := (chunk.isClient() && chunk.isWrite()) || (chunk.isServer() && chunk.isRead()) + tcpid := buildTcpId(isRequest, ip, port) + + go dissect(extension, reader, isRequest, &tcpid, emitter, options, p.reqResMatcher) + return reader +} + +func dissect(extension *api.Extension, reader *tlsReader, isRequest bool, tcpid *api.TcpID, + emitter api.Emitter, options *api.TrafficFilteringOptions, reqResMatcher api.RequestResponseMatcher) { + b := bufio.NewReader(reader) + + err := extension.Dissector.Dissect(b, isRequest, tcpid, &api.CounterPair{}, + &api.SuperTimer{}, &api.SuperIdentifier{}, emitter, options, reqResMatcher) + + if err != nil { + logger.Log.Warningf("Error dissecting TLS %v - %v", tcpid, err) + } +} + +func (p *tlsPoller) closeReader(key string, r *tlsReader) { + close(r.chunks) + p.closedReaders <- key +} + +func buildTlsKey(chunk *tlsChunk, ip net.IP, port uint16) string { + return fmt.Sprintf("%v:%v-%v:%v", chunk.isClient(), chunk.isRead(), ip, port) +} + +func buildTcpId(isRequest bool, ip net.IP, port uint16) api.TcpID { + if isRequest { + return api.TcpID{ + SrcIP: UNKNOWN_HOST, + DstIP: ip.String(), + SrcPort: strconv.Itoa(int(UNKNOWN_PORT)), + DstPort: strconv.FormatInt(int64(port), 10), + Ident: "", + } + } else { + return api.TcpID{ + SrcIP: ip.String(), + DstIP: UNKNOWN_HOST, + SrcPort: strconv.FormatInt(int64(port), 10), + DstPort: strconv.Itoa(int(UNKNOWN_PORT)), + Ident: "", + } + } +} + +func logTls(chunk *tlsChunk, ip net.IP, port uint16) { + var flagsStr string + + if chunk.isClient() { + flagsStr = "C" + } else { + flagsStr = "S" + } + + if chunk.isRead() { + flagsStr += "R" + } else { + flagsStr += "W" + } + + str := strings.ReplaceAll(strings.ReplaceAll(string(chunk.Data[0:chunk.Recorded]), "\n", " "), "\r", "") + + logger.Log.Infof("PID: %v (tid: %v) (fd: %v) (client: %v) (addr: %v:%v) (recorded %v out of %v) - %v - %v", + chunk.Pid, chunk.Tgid, chunk.Fd, flagsStr, ip, port, chunk.Recorded, chunk.Len, str, hex.EncodeToString(chunk.Data[0:chunk.Recorded])) +} diff --git a/tap/tlstapper/tls_process_discoverer.go b/tap/tlstapper/tls_process_discoverer.go new file mode 100644 index 000000000..df785d2b6 --- /dev/null +++ b/tap/tlstapper/tls_process_discoverer.go @@ -0,0 +1,143 @@ +package tlstapper + +import ( + "fmt" + "io/ioutil" + "net/url" + "path" + "path/filepath" + "regexp" + "strconv" + "strings" + + "github.com/go-errors/errors" + "github.com/up9inc/mizu/shared/logger" + v1 "k8s.io/api/core/v1" +) + +var numberRegex = regexp.MustCompile("[0-9]+") + +func UpdateTapTargets(tls *TlsTapper, pods *[]v1.Pod, procfs string) error { + containerIds := buildContainerIdsMap(pods) + containerPids, err := findContainerPids(procfs, containerIds) + + if err != nil { + return err + } + + for _, pid := range containerPids { + if err := tls.AddPid(procfs, pid); err != nil { + LogError(err) + } + } + + return nil +} + +func findContainerPids(procfs string, containerIds map[string]bool) ([]uint32, error) { + result := make([]uint32, 0) + + pids, err := ioutil.ReadDir(procfs) + + if err != nil { + return result, err + } + + logger.Log.Infof("Starting tls auto discoverer %v %v - scanning %v potential pids", + procfs, containerIds, len(pids)) + + for _, pid := range pids { + if !pid.IsDir() { + continue + } + + if !numberRegex.MatchString(pid.Name()) { + continue + } + + cgroup, err := getProcessCgroup(procfs, pid.Name()) + + if err != nil { + continue + } + + if _, ok := containerIds[cgroup]; !ok { + continue + } + + pidNumber, err := strconv.Atoi(pid.Name()) + + if err != nil { + continue + } + + result = append(result, uint32(pidNumber)) + } + + return result, nil +} + +func buildContainerIdsMap(pods *[]v1.Pod) map[string]bool { + result := make(map[string]bool) + + for _, pod := range *pods { + for _, container := range pod.Status.ContainerStatuses { + url, err := url.Parse(container.ContainerID) + + if err != nil { + logger.Log.Warningf("Expecting URL like container ID %v", container.ContainerID) + continue + } + + result[url.Host] = true + } + } + + return result +} + +func getProcessCgroup(procfs string, pid string) (string, error) { + filePath := fmt.Sprintf("%s/%s/cgroup", procfs, pid) + + bytes, err := ioutil.ReadFile(filePath) + + if err != nil { + logger.Log.Warningf("Error reading cgroup file %s - %v", filePath, err) + return "", err + } + + lines := strings.Split(string(bytes), "\n") + cgrouppath := extractCgroup(lines) + + if cgrouppath == "" { + return "", errors.Errorf("Cgroup path not found for %s, %s", pid, lines) + } + + return normalizeCgroup(cgrouppath), nil +} + +func extractCgroup(lines []string) string { + if len(lines) == 1 { + parts := strings.Split(lines[0], ":") + return parts[len(parts)-1] + } else { + for _, line := range lines { + if strings.Contains(line, ":pids:") { + parts := strings.Split(line, ":") + return parts[len(parts)-1] + } + } + } + + return "" +} + +func normalizeCgroup(cgrouppath string) string { + basename := strings.TrimSpace(path.Base(cgrouppath)) + + if strings.Contains(basename, ".") { + return strings.TrimSuffix(basename, filepath.Ext(basename)) + } else { + return basename + } +} diff --git a/tap/tlstapper/tls_reader.go b/tap/tlstapper/tls_reader.go new file mode 100644 index 000000000..59b3e7c9a --- /dev/null +++ b/tap/tlstapper/tls_reader.go @@ -0,0 +1,41 @@ +package tlstapper + +import ( + "io" + "time" +) + +type tlsReader struct { + key string + chunks chan *tlsChunk + data []byte + doneHandler func(r *tlsReader) +} + +func (r *tlsReader) Read(p []byte) (int, error) { + var chunk *tlsChunk + + for len(r.data) == 0 { + var ok bool + select { + case chunk, ok = <-r.chunks: + if !ok { + return 0, io.EOF + } + + r.data = chunk.getRecordedData() + case <-time.After(time.Second * 3): + r.doneHandler(r) + return 0, io.EOF + } + + if len(r.data) > 0 { + break + } + } + + l := copy(p, r.data) + r.data = r.data[l:] + + return l, nil +} diff --git a/tap/tlstapper/tls_tapper.go b/tap/tlstapper/tls_tapper.go new file mode 100644 index 000000000..4be96a994 --- /dev/null +++ b/tap/tlstapper/tls_tapper.go @@ -0,0 +1,177 @@ +package tlstapper + +import ( + "bytes" + "encoding/binary" + + "github.com/cilium/ebpf/perf" + "github.com/cilium/ebpf/rlimit" + "github.com/go-errors/errors" + "github.com/up9inc/mizu/shared/logger" +) + +//go:generate go run github.com/cilium/ebpf/cmd/bpf2go tlsTapper bpf/tls_tapper.c -- -O2 -g -D__TARGET_ARCH_x86 + +type TlsTapper struct { + bpfObjects tlsTapperObjects + syscallHooks syscallHooks + sslHooksStructs []sslHooks + reader *perf.Reader +} + +func (t *TlsTapper) Init(bufferSize int) error { + logger.Log.Infof("Initializing tls tapper (bufferSize: %v)", bufferSize) + + if err := setupRLimit(); err != nil { + return err + } + + t.bpfObjects = tlsTapperObjects{} + + if err := loadTlsTapperObjects(&t.bpfObjects, nil); err != nil { + return errors.Wrap(err, 0) + } + + t.syscallHooks = syscallHooks{} + + if err := t.syscallHooks.installSyscallHooks(&t.bpfObjects); err != nil { + return err + } + + t.sslHooksStructs = make([]sslHooks, 0) + + return t.initChunksReader(bufferSize) +} + +func (t *TlsTapper) pollPerf(chunks chan<- *tlsChunk) { + logger.Log.Infof("Start polling for tls events") + + for { + record, err := t.reader.Read() + + if err != nil { + close(chunks) + + if errors.Is(err, perf.ErrClosed) { + return + } + + LogError(errors.Errorf("Error reading chunks from tls perf, aborting TLS! %v", err)) + return + } + + if record.LostSamples != 0 { + logger.Log.Infof("Buffer is full, dropped %d chunks", record.LostSamples) + continue + } + + buffer := bytes.NewReader(record.RawSample) + + var chunk tlsChunk + + if err := binary.Read(buffer, binary.LittleEndian, &chunk); err != nil { + LogError(errors.Errorf("Error parsing chunk %v", err)) + continue + } + + chunks <- &chunk + } +} + +func (t *TlsTapper) GlobalTap(sslLibrary string) error { + return t.tapPid(0, sslLibrary) +} + +func (t *TlsTapper) AddPid(procfs string, pid uint32) error { + sslLibrary, err := findSsllib(procfs, pid) + + if err != nil { + logger.Log.Infof("PID skipped no libssl.so found (pid: %d) %v", pid, err) + return nil // hide the error on purpose, its OK for a process to not use libssl.so + } + + return t.tapPid(pid, sslLibrary) +} + +func (t *TlsTapper) RemovePid(pid uint32) error { + logger.Log.Infof("Removing PID (pid: %v)", pid) + + pids := t.bpfObjects.tlsTapperMaps.PidsMap + + if err := pids.Delete(pid); err != nil { + return errors.Wrap(err, 0) + } + + return nil +} + +func (t *TlsTapper) Close() []error { + errors := make([]error, 0) + + if err := t.bpfObjects.Close(); err != nil { + errors = append(errors, err) + } + + errors = append(errors, t.syscallHooks.close()...) + + for _, sslHooks := range t.sslHooksStructs { + errors = append(errors, sslHooks.close()...) + } + + if err := t.reader.Close(); err != nil { + errors = append(errors, err) + } + + return errors +} + +func setupRLimit() error { + err := rlimit.RemoveMemlock() + + if err != nil { + return errors.Wrap(err, 0) + } + + return nil +} + +func (t *TlsTapper) initChunksReader(bufferSize int) error { + var err error + + t.reader, err = perf.NewReader(t.bpfObjects.ChunksBuffer, bufferSize) + + if err != nil { + return errors.Wrap(err, 0) + } + + return nil +} + +func (t *TlsTapper) tapPid(pid uint32, sslLibrary string) error { + logger.Log.Infof("Tapping TLS (pid: %v) (sslLibrary: %v)", pid, sslLibrary) + + newSsl := sslHooks{} + + if err := newSsl.installUprobes(&t.bpfObjects, sslLibrary); err != nil { + return err + } + + t.sslHooksStructs = append(t.sslHooksStructs, newSsl) + + pids := t.bpfObjects.tlsTapperMaps.PidsMap + + if err := pids.Put(pid, uint32(1)); err != nil { + return errors.Wrap(err, 0) + } + + return nil +} + +func LogError(err error) { + var e *errors.Error + if errors.As(err, &e) { + logger.Log.Errorf("Error: %v", e.ErrorStack()) + } else { + logger.Log.Errorf("Error: %v", err) + } +} diff --git a/tap/tlstapper/tlstapper_bpfeb.go b/tap/tlstapper/tlstapper_bpfeb.go new file mode 100644 index 000000000..465966801 --- /dev/null +++ b/tap/tlstapper/tlstapper_bpfeb.go @@ -0,0 +1,179 @@ +// Code generated by bpf2go; DO NOT EDIT. +//go:build arm64be || armbe || mips || mips64 || mips64p32 || ppc64 || s390 || s390x || sparc || sparc64 +// +build arm64be armbe mips mips64 mips64p32 ppc64 s390 s390x sparc sparc64 + +package tlstapper + +import ( + "bytes" + _ "embed" + "fmt" + "io" + + "github.com/cilium/ebpf" +) + +// loadTlsTapper returns the embedded CollectionSpec for tlsTapper. +func loadTlsTapper() (*ebpf.CollectionSpec, error) { + reader := bytes.NewReader(_TlsTapperBytes) + spec, err := ebpf.LoadCollectionSpecFromReader(reader) + if err != nil { + return nil, fmt.Errorf("can't load tlsTapper: %w", err) + } + + return spec, err +} + +// loadTlsTapperObjects loads tlsTapper and converts it into a struct. +// +// The following types are suitable as obj argument: +// +// *tlsTapperObjects +// *tlsTapperPrograms +// *tlsTapperMaps +// +// See ebpf.CollectionSpec.LoadAndAssign documentation for details. +func loadTlsTapperObjects(obj interface{}, opts *ebpf.CollectionOptions) error { + spec, err := loadTlsTapper() + if err != nil { + return err + } + + return spec.LoadAndAssign(obj, opts) +} + +// tlsTapperSpecs contains maps and programs before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type tlsTapperSpecs struct { + tlsTapperProgramSpecs + tlsTapperMapSpecs +} + +// tlsTapperSpecs contains programs before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type tlsTapperProgramSpecs struct { + SslRead *ebpf.ProgramSpec `ebpf:"ssl_read"` + SslReadEx *ebpf.ProgramSpec `ebpf:"ssl_read_ex"` + SslRetRead *ebpf.ProgramSpec `ebpf:"ssl_ret_read"` + SslRetReadEx *ebpf.ProgramSpec `ebpf:"ssl_ret_read_ex"` + SslRetWrite *ebpf.ProgramSpec `ebpf:"ssl_ret_write"` + SslRetWriteEx *ebpf.ProgramSpec `ebpf:"ssl_ret_write_ex"` + SslWrite *ebpf.ProgramSpec `ebpf:"ssl_write"` + SslWriteEx *ebpf.ProgramSpec `ebpf:"ssl_write_ex"` + SysEnterAccept4 *ebpf.ProgramSpec `ebpf:"sys_enter_accept4"` + SysEnterConnect *ebpf.ProgramSpec `ebpf:"sys_enter_connect"` + SysEnterRead *ebpf.ProgramSpec `ebpf:"sys_enter_read"` + SysEnterWrite *ebpf.ProgramSpec `ebpf:"sys_enter_write"` + SysExitAccept4 *ebpf.ProgramSpec `ebpf:"sys_exit_accept4"` + SysExitConnect *ebpf.ProgramSpec `ebpf:"sys_exit_connect"` +} + +// tlsTapperMapSpecs contains maps before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type tlsTapperMapSpecs struct { + AcceptSyscallContext *ebpf.MapSpec `ebpf:"accept_syscall_context"` + ChunksBuffer *ebpf.MapSpec `ebpf:"chunks_buffer"` + ConnectSyscallInfo *ebpf.MapSpec `ebpf:"connect_syscall_info"` + FileDescriptorToIpv4 *ebpf.MapSpec `ebpf:"file_descriptor_to_ipv4"` + Heap *ebpf.MapSpec `ebpf:"heap"` + PidsMap *ebpf.MapSpec `ebpf:"pids_map"` + SslReadContext *ebpf.MapSpec `ebpf:"ssl_read_context"` + SslWriteContext *ebpf.MapSpec `ebpf:"ssl_write_context"` +} + +// tlsTapperObjects contains all objects after they have been loaded into the kernel. +// +// It can be passed to loadTlsTapperObjects or ebpf.CollectionSpec.LoadAndAssign. +type tlsTapperObjects struct { + tlsTapperPrograms + tlsTapperMaps +} + +func (o *tlsTapperObjects) Close() error { + return _TlsTapperClose( + &o.tlsTapperPrograms, + &o.tlsTapperMaps, + ) +} + +// tlsTapperMaps contains all maps after they have been loaded into the kernel. +// +// It can be passed to loadTlsTapperObjects or ebpf.CollectionSpec.LoadAndAssign. +type tlsTapperMaps struct { + AcceptSyscallContext *ebpf.Map `ebpf:"accept_syscall_context"` + ChunksBuffer *ebpf.Map `ebpf:"chunks_buffer"` + ConnectSyscallInfo *ebpf.Map `ebpf:"connect_syscall_info"` + FileDescriptorToIpv4 *ebpf.Map `ebpf:"file_descriptor_to_ipv4"` + Heap *ebpf.Map `ebpf:"heap"` + PidsMap *ebpf.Map `ebpf:"pids_map"` + SslReadContext *ebpf.Map `ebpf:"ssl_read_context"` + SslWriteContext *ebpf.Map `ebpf:"ssl_write_context"` +} + +func (m *tlsTapperMaps) Close() error { + return _TlsTapperClose( + m.AcceptSyscallContext, + m.ChunksBuffer, + m.ConnectSyscallInfo, + m.FileDescriptorToIpv4, + m.Heap, + m.PidsMap, + m.SslReadContext, + m.SslWriteContext, + ) +} + +// tlsTapperPrograms contains all programs after they have been loaded into the kernel. +// +// It can be passed to loadTlsTapperObjects or ebpf.CollectionSpec.LoadAndAssign. +type tlsTapperPrograms struct { + SslRead *ebpf.Program `ebpf:"ssl_read"` + SslReadEx *ebpf.Program `ebpf:"ssl_read_ex"` + SslRetRead *ebpf.Program `ebpf:"ssl_ret_read"` + SslRetReadEx *ebpf.Program `ebpf:"ssl_ret_read_ex"` + SslRetWrite *ebpf.Program `ebpf:"ssl_ret_write"` + SslRetWriteEx *ebpf.Program `ebpf:"ssl_ret_write_ex"` + SslWrite *ebpf.Program `ebpf:"ssl_write"` + SslWriteEx *ebpf.Program `ebpf:"ssl_write_ex"` + SysEnterAccept4 *ebpf.Program `ebpf:"sys_enter_accept4"` + SysEnterConnect *ebpf.Program `ebpf:"sys_enter_connect"` + SysEnterRead *ebpf.Program `ebpf:"sys_enter_read"` + SysEnterWrite *ebpf.Program `ebpf:"sys_enter_write"` + SysExitAccept4 *ebpf.Program `ebpf:"sys_exit_accept4"` + SysExitConnect *ebpf.Program `ebpf:"sys_exit_connect"` +} + +func (p *tlsTapperPrograms) Close() error { + return _TlsTapperClose( + p.SslRead, + p.SslReadEx, + p.SslRetRead, + p.SslRetReadEx, + p.SslRetWrite, + p.SslRetWriteEx, + p.SslWrite, + p.SslWriteEx, + p.SysEnterAccept4, + p.SysEnterConnect, + p.SysEnterRead, + p.SysEnterWrite, + p.SysExitAccept4, + p.SysExitConnect, + ) +} + +func _TlsTapperClose(closers ...io.Closer) error { + for _, closer := range closers { + if err := closer.Close(); err != nil { + return err + } + } + return nil +} + +// Do not access this directly. +//go:embed tlstapper_bpfeb.o +var _TlsTapperBytes []byte diff --git a/tap/tlstapper/tlstapper_bpfeb.o b/tap/tlstapper/tlstapper_bpfeb.o new file mode 100644 index 000000000..e09017a41 Binary files /dev/null and b/tap/tlstapper/tlstapper_bpfeb.o differ diff --git a/tap/tlstapper/tlstapper_bpfel.go b/tap/tlstapper/tlstapper_bpfel.go new file mode 100644 index 000000000..7e962bbb0 --- /dev/null +++ b/tap/tlstapper/tlstapper_bpfel.go @@ -0,0 +1,179 @@ +// Code generated by bpf2go; DO NOT EDIT. +//go:build 386 || amd64 || amd64p32 || arm || arm64 || mips64le || mips64p32le || mipsle || ppc64le || riscv64 +// +build 386 amd64 amd64p32 arm arm64 mips64le mips64p32le mipsle ppc64le riscv64 + +package tlstapper + +import ( + "bytes" + _ "embed" + "fmt" + "io" + + "github.com/cilium/ebpf" +) + +// loadTlsTapper returns the embedded CollectionSpec for tlsTapper. +func loadTlsTapper() (*ebpf.CollectionSpec, error) { + reader := bytes.NewReader(_TlsTapperBytes) + spec, err := ebpf.LoadCollectionSpecFromReader(reader) + if err != nil { + return nil, fmt.Errorf("can't load tlsTapper: %w", err) + } + + return spec, err +} + +// loadTlsTapperObjects loads tlsTapper and converts it into a struct. +// +// The following types are suitable as obj argument: +// +// *tlsTapperObjects +// *tlsTapperPrograms +// *tlsTapperMaps +// +// See ebpf.CollectionSpec.LoadAndAssign documentation for details. +func loadTlsTapperObjects(obj interface{}, opts *ebpf.CollectionOptions) error { + spec, err := loadTlsTapper() + if err != nil { + return err + } + + return spec.LoadAndAssign(obj, opts) +} + +// tlsTapperSpecs contains maps and programs before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type tlsTapperSpecs struct { + tlsTapperProgramSpecs + tlsTapperMapSpecs +} + +// tlsTapperSpecs contains programs before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type tlsTapperProgramSpecs struct { + SslRead *ebpf.ProgramSpec `ebpf:"ssl_read"` + SslReadEx *ebpf.ProgramSpec `ebpf:"ssl_read_ex"` + SslRetRead *ebpf.ProgramSpec `ebpf:"ssl_ret_read"` + SslRetReadEx *ebpf.ProgramSpec `ebpf:"ssl_ret_read_ex"` + SslRetWrite *ebpf.ProgramSpec `ebpf:"ssl_ret_write"` + SslRetWriteEx *ebpf.ProgramSpec `ebpf:"ssl_ret_write_ex"` + SslWrite *ebpf.ProgramSpec `ebpf:"ssl_write"` + SslWriteEx *ebpf.ProgramSpec `ebpf:"ssl_write_ex"` + SysEnterAccept4 *ebpf.ProgramSpec `ebpf:"sys_enter_accept4"` + SysEnterConnect *ebpf.ProgramSpec `ebpf:"sys_enter_connect"` + SysEnterRead *ebpf.ProgramSpec `ebpf:"sys_enter_read"` + SysEnterWrite *ebpf.ProgramSpec `ebpf:"sys_enter_write"` + SysExitAccept4 *ebpf.ProgramSpec `ebpf:"sys_exit_accept4"` + SysExitConnect *ebpf.ProgramSpec `ebpf:"sys_exit_connect"` +} + +// tlsTapperMapSpecs contains maps before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type tlsTapperMapSpecs struct { + AcceptSyscallContext *ebpf.MapSpec `ebpf:"accept_syscall_context"` + ChunksBuffer *ebpf.MapSpec `ebpf:"chunks_buffer"` + ConnectSyscallInfo *ebpf.MapSpec `ebpf:"connect_syscall_info"` + FileDescriptorToIpv4 *ebpf.MapSpec `ebpf:"file_descriptor_to_ipv4"` + Heap *ebpf.MapSpec `ebpf:"heap"` + PidsMap *ebpf.MapSpec `ebpf:"pids_map"` + SslReadContext *ebpf.MapSpec `ebpf:"ssl_read_context"` + SslWriteContext *ebpf.MapSpec `ebpf:"ssl_write_context"` +} + +// tlsTapperObjects contains all objects after they have been loaded into the kernel. +// +// It can be passed to loadTlsTapperObjects or ebpf.CollectionSpec.LoadAndAssign. +type tlsTapperObjects struct { + tlsTapperPrograms + tlsTapperMaps +} + +func (o *tlsTapperObjects) Close() error { + return _TlsTapperClose( + &o.tlsTapperPrograms, + &o.tlsTapperMaps, + ) +} + +// tlsTapperMaps contains all maps after they have been loaded into the kernel. +// +// It can be passed to loadTlsTapperObjects or ebpf.CollectionSpec.LoadAndAssign. +type tlsTapperMaps struct { + AcceptSyscallContext *ebpf.Map `ebpf:"accept_syscall_context"` + ChunksBuffer *ebpf.Map `ebpf:"chunks_buffer"` + ConnectSyscallInfo *ebpf.Map `ebpf:"connect_syscall_info"` + FileDescriptorToIpv4 *ebpf.Map `ebpf:"file_descriptor_to_ipv4"` + Heap *ebpf.Map `ebpf:"heap"` + PidsMap *ebpf.Map `ebpf:"pids_map"` + SslReadContext *ebpf.Map `ebpf:"ssl_read_context"` + SslWriteContext *ebpf.Map `ebpf:"ssl_write_context"` +} + +func (m *tlsTapperMaps) Close() error { + return _TlsTapperClose( + m.AcceptSyscallContext, + m.ChunksBuffer, + m.ConnectSyscallInfo, + m.FileDescriptorToIpv4, + m.Heap, + m.PidsMap, + m.SslReadContext, + m.SslWriteContext, + ) +} + +// tlsTapperPrograms contains all programs after they have been loaded into the kernel. +// +// It can be passed to loadTlsTapperObjects or ebpf.CollectionSpec.LoadAndAssign. +type tlsTapperPrograms struct { + SslRead *ebpf.Program `ebpf:"ssl_read"` + SslReadEx *ebpf.Program `ebpf:"ssl_read_ex"` + SslRetRead *ebpf.Program `ebpf:"ssl_ret_read"` + SslRetReadEx *ebpf.Program `ebpf:"ssl_ret_read_ex"` + SslRetWrite *ebpf.Program `ebpf:"ssl_ret_write"` + SslRetWriteEx *ebpf.Program `ebpf:"ssl_ret_write_ex"` + SslWrite *ebpf.Program `ebpf:"ssl_write"` + SslWriteEx *ebpf.Program `ebpf:"ssl_write_ex"` + SysEnterAccept4 *ebpf.Program `ebpf:"sys_enter_accept4"` + SysEnterConnect *ebpf.Program `ebpf:"sys_enter_connect"` + SysEnterRead *ebpf.Program `ebpf:"sys_enter_read"` + SysEnterWrite *ebpf.Program `ebpf:"sys_enter_write"` + SysExitAccept4 *ebpf.Program `ebpf:"sys_exit_accept4"` + SysExitConnect *ebpf.Program `ebpf:"sys_exit_connect"` +} + +func (p *tlsTapperPrograms) Close() error { + return _TlsTapperClose( + p.SslRead, + p.SslReadEx, + p.SslRetRead, + p.SslRetReadEx, + p.SslRetWrite, + p.SslRetWriteEx, + p.SslWrite, + p.SslWriteEx, + p.SysEnterAccept4, + p.SysEnterConnect, + p.SysEnterRead, + p.SysEnterWrite, + p.SysExitAccept4, + p.SysExitConnect, + ) +} + +func _TlsTapperClose(closers ...io.Closer) error { + for _, closer := range closers { + if err := closer.Close(); err != nil { + return err + } + } + return nil +} + +// Do not access this directly. +//go:embed tlstapper_bpfel.o +var _TlsTapperBytes []byte diff --git a/tap/tlstapper/tlstapper_bpfel.o b/tap/tlstapper/tlstapper_bpfel.o new file mode 100644 index 000000000..607ea07e4 Binary files /dev/null and b/tap/tlstapper/tlstapper_bpfel.o differ