From f5d87c3d94fb0ccae6a45cd1cdc4e38965699e4e Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Fri, 5 Sep 2025 15:06:26 -0400 Subject: [PATCH 01/75] draft: create v04 and v1 with interface --- ddtrace/tracer/payload.go | 189 ++------------------------------- ddtrace/tracer/payload_test.go | 4 +- ddtrace/tracer/payload_v04.go | 179 +++++++++++++++++++++++++++++++ ddtrace/tracer/payload_v1.go | 179 +++++++++++++++++++++++++++++++ 4 files changed, 370 insertions(+), 181 deletions(-) create mode 100644 ddtrace/tracer/payload_v04.go create mode 100644 ddtrace/tracer/payload_v1.go diff --git a/ddtrace/tracer/payload.go b/ddtrace/tracer/payload.go index 3cb58ef18a..5730d37315 100644 --- a/ddtrace/tracer/payload.go +++ b/ddtrace/tracer/payload.go @@ -6,13 +6,8 @@ package tracer import ( - "bytes" - "encoding/binary" "io" "sync" - "sync/atomic" - - "github.com/tinylib/msgp/msgp" ) // payloadStats contains the statistics of a payload. @@ -51,103 +46,16 @@ type payload interface { payloadReader } -// unsafePayload is a wrapper on top of the msgpack encoder which allows constructing an -// encoded array by pushing its entries sequentially, one at a time. It basically -// allows us to encode as we would with a stream, except that the contents of the stream -// can be read as a slice by the msgpack decoder at any time. It follows the guidelines -// from the msgpack array spec: -// https://github.com/msgpack/msgpack/blob/master/spec.md#array-format-family -// -// unsafePayload implements io.Reader and can be used with the decoder directly. -// -// unsafePayload is not safe for concurrent use. -// -// unsafePayload is meant to be used only once and eventually dismissed with the -// single exception of retrying failed flush attempts. -// -// ⚠️ Warning! -// -// The payload should not be reused for multiple sets of traces. Resetting the -// payload for re-use requires the transport to wait for the HTTP package to -// Close the request body before attempting to re-use it again! This requires -// additional logic to be in place. See: -// -// • https://github.com/golang/go/blob/go1.16/src/net/http/client.go#L136-L138 -// • https://github.com/DataDog/dd-trace-go/pull/475 -// • https://github.com/DataDog/dd-trace-go/pull/549 -// • https://github.com/DataDog/dd-trace-go/pull/976 -type unsafePayload struct { - // header specifies the first few bytes in the msgpack stream - // indicating the type of array (fixarray, array16 or array32) - // and the number of items contained in the stream. - header []byte - - // off specifies the current read position on the header. - off int - - // count specifies the number of items in the stream. - count uint32 - - // buf holds the sequence of msgpack-encoded items. - buf bytes.Buffer - - // reader is used for reading the contents of buf. - reader *bytes.Reader - - // protocolVersion specifies the trace protocolVersion to use. - protocolVersion float64 -} - -var _ io.Reader = (*unsafePayload)(nil) - -// newUnsafePayload returns a ready to use unsafe payload. -func newUnsafePayload(protocol float64) *unsafePayload { - p := &unsafePayload{ - header: make([]byte, 8), - off: 8, - protocolVersion: protocol, - } - return p -} - -// push pushes a new item into the stream. -func (p *unsafePayload) push(t []*Span) (stats payloadStats, err error) { - sl := spanList(t) - p.buf.Grow(sl.Msgsize()) - if err := msgp.Encode(&p.buf, sl); err != nil { - return payloadStats{}, err - } - p.recordItem() - return p.stats(), nil -} - -// itemCount returns the number of items available in the stream. -func (p *unsafePayload) itemCount() int { - return int(atomic.LoadUint32(&p.count)) -} - -// size returns the payload size in bytes. After the first read the value becomes -// inaccurate by up to 8 bytes. -func (p *unsafePayload) size() int { - return p.buf.Len() + len(p.header) - p.off -} - -// reset sets up the payload to be read a second time. It maintains the -// underlying byte contents of the buffer. reset should not be used in order to -// reuse the payload for another set of traces. -func (p *unsafePayload) reset() { - p.updateHeader() - if p.reader != nil { - p.reader.Seek(0, 0) +// newpayload returns a ready to use unsafe payload. +func newPayload(protocol float64) payload { + // TODO(hannahkm): add support for v1 protocol + // if protocol == traceProtocolV1 { + // } + return &safePayload{ + p: newPayloadV04(), } } -// clear empties the payload buffers. -func (p *unsafePayload) clear() { - p.buf = bytes.Buffer{} - p.reader = nil -} - // https://github.com/msgpack/msgpack/blob/master/spec.md#array-format-family const ( msgpackArrayFix byte = 144 // up to 15 items @@ -155,86 +63,10 @@ const ( msgpackArray32 byte = 0xdd // up to 2^32-1 items, followed by size in 4 bytes ) -// updateHeader updates the payload header based on the number of items currently -// present in the stream. -func (p *unsafePayload) updateHeader() { - n := uint64(atomic.LoadUint32(&p.count)) - switch { - case n <= 15: - p.header[7] = msgpackArrayFix + byte(n) - p.off = 7 - case n <= 1<<16-1: - binary.BigEndian.PutUint64(p.header, n) // writes 2 bytes - p.header[5] = msgpackArray16 - p.off = 5 - default: // n <= 1<<32-1 - binary.BigEndian.PutUint64(p.header, n) // writes 4 bytes - p.header[3] = msgpackArray32 - p.off = 3 - } -} - -// Close implements io.Closer -func (p *unsafePayload) Close() error { - return nil -} - -// Read implements io.Reader. It reads from the msgpack-encoded stream. -func (p *unsafePayload) Read(b []byte) (n int, err error) { - if p.off < len(p.header) { - // reading header - n = copy(b, p.header[p.off:]) - p.off += n - return n, nil - } - if p.reader == nil { - p.reader = bytes.NewReader(p.buf.Bytes()) - } - return p.reader.Read(b) -} - -// Write implements io.Writer. It writes data directly to the buffer. -func (p *unsafePayload) Write(data []byte) (n int, err error) { - return p.buf.Write(data) -} - -// grow grows the buffer to ensure it can accommodate n more bytes. -func (p *unsafePayload) grow(n int) { - p.buf.Grow(n) -} - -// recordItem records that an item was added and updates the header. -func (p *unsafePayload) recordItem() { - atomic.AddUint32(&p.count, 1) - p.updateHeader() -} - -// stats returns the current stats of the payload. -func (p *unsafePayload) stats() payloadStats { - return payloadStats{ - size: p.size(), - itemCount: int(atomic.LoadUint32(&p.count)), - } -} - -// protocol returns the protocol version of the payload. -func (p *unsafePayload) protocol() float64 { - return p.protocolVersion -} - -var _ io.Reader = (*safePayload)(nil) - -// newPayload returns a ready to use thread-safe payload. -func newPayload(protocol float64) payload { - return &safePayload{ - p: newUnsafePayload(protocol), - } -} - -// safePayload provides a thread-safe wrapper around unsafePayload. +// safePayload provides a thread-safe wrapper around payload. type safePayload struct { mu sync.RWMutex - p *unsafePayload + p payload } // push pushes a new item into the stream in a thread-safe manner. @@ -246,8 +78,7 @@ func (sp *safePayload) push(t spanList) (stats payloadStats, err error) { // itemCount returns the number of items available in the stream in a thread-safe manner. func (sp *safePayload) itemCount() int { - // Use direct atomic access for better performance - no mutex needed - return int(atomic.LoadUint32(&sp.p.count)) + return sp.p.itemCount() } // size returns the payload size in bytes in a thread-safe manner. diff --git a/ddtrace/tracer/payload_test.go b/ddtrace/tracer/payload_test.go index e860a7f9f4..239ca52385 100644 --- a/ddtrace/tracer/payload_test.go +++ b/ddtrace/tracer/payload_test.go @@ -88,7 +88,7 @@ func BenchmarkPayloadThroughput(b *testing.B) { // payload is filled. func benchmarkPayloadThroughput(count int) func(*testing.B) { return func(b *testing.B) { - p := newUnsafePayload(traceProtocolV04) + p := newPayloadV04() s := newBasicSpan("X") s.meta["key"] = strings.Repeat("X", 10*1024) trace := make(spanList, count) @@ -248,7 +248,7 @@ func BenchmarkPayloadPush(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - p := newUnsafePayload(traceProtocolV04) + p := newPayloadV04() _, _ = p.push(spans) } }) diff --git a/ddtrace/tracer/payload_v04.go b/ddtrace/tracer/payload_v04.go new file mode 100644 index 0000000000..54c99b1800 --- /dev/null +++ b/ddtrace/tracer/payload_v04.go @@ -0,0 +1,179 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2025 Datadog, Inc. + +package tracer + +import ( + "bytes" + "encoding/binary" + "io" + "sync/atomic" + + "github.com/tinylib/msgp/msgp" +) + +// payloadV04 is a wrapper on top of the msgpack encoder which allows constructing an +// encoded array by pushing its entries sequentially, one at a time. It basically +// allows us to encode as we would with a stream, except that the contents of the stream +// can be read as a slice by the msgpack decoder at any time. It follows the guidelines +// from the msgpack array spec: +// https://github.com/msgpack/msgpack/blob/master/spec.md#array-format-family +// +// payloadV04 implements unsafePayload and can be used with the decoder directly. To create +// a new payload use the newPayloadV04 method. +// +// payloadV04 is not safe for concurrent use. +// +// payloadV04 is meant to be used only once and eventually dismissed with the +// single exception of retrying failed flush attempts. +// +// ⚠️ Warning! +// +// The payloadV04 should not be reused for multiple sets of traces. Resetting the +// payloadV04 for re-use requires the transport to wait for the HTTP package to +// Close the request body before attempting to re-use it again! This requires +// additional logic to be in place. See: +// +// • https://github.com/golang/go/blob/go1.16/src/net/http/client.go#L136-L138 +// • https://github.com/DataDog/dd-trace-go/pull/475 +// • https://github.com/DataDog/dd-trace-go/pull/549 +// • https://github.com/DataDog/dd-trace-go/pull/976 +type payloadV04 struct { + // header specifies the first few bytes in the msgpack stream + // indicating the type of array (fixarray, array16 or array32) + // and the number of items contained in the stream. + header []byte + + // off specifies the current read position on the header. + off int + + // count specifies the number of items in the stream. + count uint32 + + // buf holds the sequence of msgpack-encoded items. + buf bytes.Buffer + + // reader is used for reading the contents of buf. + reader *bytes.Reader + + // protocolVersion specifies the trace protocol to use. + protocolVersion float64 +} + +var _ io.Reader = (*payloadV04)(nil) + +// newPayloadV04 returns a ready to use payload. +func newPayloadV04() *payloadV04 { + p := &payloadV04{ + header: make([]byte, 8), + off: 8, + protocolVersion: traceProtocolV04, + } + return p +} + +// push pushes a new item into the stream. +func (p *payloadV04) push(t spanList) (stats payloadStats, err error) { + p.buf.Grow(t.Msgsize()) + if err := msgp.Encode(&p.buf, t); err != nil { + return payloadStats{}, err + } + p.recordItem() + return p.stats(), nil +} + +// itemCount returns the number of items available in the stream. +func (p *payloadV04) itemCount() int { + return int(atomic.LoadUint32(&p.count)) +} + +// size returns the payload size in bytes. After the first read the value becomes +// inaccurate by up to 8 bytes. +func (p *payloadV04) size() int { + return p.buf.Len() + len(p.header) - p.off +} + +// reset sets up the payload to be read a second time. It maintains the +// underlying byte contents of the buffer. reset should not be used in order to +// reuse the payload for another set of traces. +func (p *payloadV04) reset() { + p.updateHeader() + if p.reader != nil { + p.reader.Seek(0, 0) + } +} + +// clear empties the payload buffers. +func (p *payloadV04) clear() { + p.buf = bytes.Buffer{} + p.reader = nil +} + +// grow grows the buffer to ensure it can accommodate n more bytes. +func (p *payloadV04) grow(n int) { + p.buf.Grow(n) +} + +// recordItem records that an item was added and updates the header. +func (p *payloadV04) recordItem() { + atomic.AddUint32(&p.count, 1) + p.updateHeader() +} + +// stats returns the current stats of the payload. +func (p *payloadV04) stats() payloadStats { + return payloadStats{ + size: p.size(), + itemCount: int(atomic.LoadUint32(&p.count)), + } +} + +// protocol returns the protocol version of the payload. +func (p *payloadV04) protocol() float64 { + return p.protocolVersion +} + +// updateHeader updates the payload header based on the number of items currently +// present in the stream. +func (p *payloadV04) updateHeader() { + n := uint64(atomic.LoadUint32(&p.count)) + switch { + case n <= 15: + p.header[7] = msgpackArrayFix + byte(n) + p.off = 7 + case n <= 1<<16-1: + binary.BigEndian.PutUint64(p.header, n) // writes 2 bytes + p.header[5] = msgpackArray16 + p.off = 5 + default: // n <= 1<<32-1 + binary.BigEndian.PutUint64(p.header, n) // writes 4 bytes + p.header[3] = msgpackArray32 + p.off = 3 + } +} + +// Close implements io.Closer +func (p *payloadV04) Close() error { + return nil +} + +// Write implements io.Writer. It writes data directly to the buffer. +func (p *payloadV04) Write(data []byte) (n int, err error) { + return p.buf.Write(data) +} + +// Read implements io.Reader. It reads from the msgpack-encoded stream. +func (p *payloadV04) Read(b []byte) (n int, err error) { + if p.off < len(p.header) { + // reading header + n = copy(b, p.header[p.off:]) + p.off += n + return n, nil + } + if p.reader == nil { + p.reader = bytes.NewReader(p.buf.Bytes()) + } + return p.reader.Read(b) +} diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go new file mode 100644 index 0000000000..38657baf5e --- /dev/null +++ b/ddtrace/tracer/payload_v1.go @@ -0,0 +1,179 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2025 Datadog, Inc. + +package tracer + +// payloadV1 is a new version of a msgp payload that can be sent to the agent. +// Be aware that payloadV1 follows the same rules and constraints as payloadV04. That is: +// +// payloadV1 is not safe for concurrent use +// +// payloadV1 is meant to be used only once and eventually dismissed with the +// single exception of retrying failed flush attempts. +// +// ⚠️ Warning! +// +// The payloadV1 should not be reused for multiple sets of traces. Resetting the +// payloadV1 for re-use requires the transport to wait for the HTTP package +// Close the request body before attempting to re-use it again! +type payloadV1 struct { + // array of strings referenced in this tracer payload, its chunks and spans + strings []string + + // the string ID of the container where the tracer is running + containerID uint32 + + // the string language name of the tracer + languageName uint32 + + // the string language version of the tracer + languageVersion uint32 + + // the string version of the tracer + tracerVersion uint32 + + // the V4 string UUID representation of a tracer session + runtimeID uint32 + + // the optional `env` string tag that set with the tracer + env uint32 + + // the optional string hostname of where the tracer is running + hostname uint32 + + // the optional string `version` tag for the application set in the tracer + appVersion uint32 + + // a collection of key to value pairs common in all `chunks` + attributes map[uint32]anyValue + + // a list of trace `chunks` + chunks []traceChunk + + // fields needed to implement unsafePayload interface + protocolVersion float64 + itemsCount uint32 +} + +// AnyValue is a representation of the `any` value. It can take the following types: +// - uint32 +// - bool +// - float64 +// - uint64 +// - uint8 +// intValue(5) - 0x405 (4 indicates this is an int AnyType, then 5 is encoded using positive fixed int format) +// stringValue(“a”) - 0x1a161 (1 indicates this is a string, then “a” is encoded using fixstr 0xa161) +// stringValue(2) - 0x102 (1 indicates this is a string, then a positive fixed int of 2 refers the 2nd index of the string table) +type anyValue struct { + valueType int + value interface{} +} + +const ( + StringValueType = iota + 1 // string or uint + BoolValueType // boolean + FloatValueType // float64 + IntValueType // uint64 + BytesValueType // []uint8 + ArrayValueType // []AnyValue + keyValueListType // []keyValue +) + +type arrayValue = []anyValue + +// keyValue is made up of the key and an AnyValue (the type of the value and the value itself) +type keyValue struct { + key uint32 + value anyValue +} + +type keyValueList = []keyValue + +// newPayloadV1 returns a ready to use payloadV1. +func newPayloadV1(protocol float64) *payloadV1 { + return &payloadV1{ + protocolVersion: protocol, + strings: make([]string, 0), + attributes: make(map[uint32]anyValue), + chunks: make([]traceChunk, 0), + } +} + +// traceChunk represents a list of spans with the same trace ID, +// i.e. a chunk of a trace +type traceChunk struct { + // the sampling priority of the trace + priority int32 + + // the optional string origin ("lambda", "rum", etc.) of the trace chunk + origin uint32 + + // a collection of key to value pairs common in all `spans` + attributes map[uint32]anyValue + + // a list of spans in this chunk + spans []Span + + // whether the trace only contains analyzed spans + // (not required by tracers and set by the agent) + droppedTrace bool + + // the ID of the trace to which all spans in this chunk belong + traceID uint8 + + // the optional string decision maker (previously span tag _dd.p.dm) + decisionMaker uint32 +} + +func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { + panic("not implemented") +} + +func (p *payloadV1) grow(n int) { + panic("not implemented") +} + +func (p *payloadV1) reset() { + panic("not implemented") +} + +func (p *payloadV1) clear() { + panic("not implemented") +} + +func (p *payloadV1) recordItem() { + panic("not implemented") +} + +func (p *payloadV1) stats() payloadStats { + panic("not implemented") +} + +func (p *payloadV1) size() int { + panic("not implemented") +} + +func (p *payloadV1) itemCount() int { + panic("not implemented") +} + +func (p *payloadV1) protocol() float64 { + panic("not implemented") +} + +// Close implements io.Closer +func (p *payloadV1) Close() error { + panic("not implemented") +} + +// Write implements io.Writer. It writes data directly to the buffer. +func (p *payloadV1) Write(data []byte) (n int, err error) { + panic("not implemented") +} + +// Read implements io.Reader. It reads from the msgpack-encoded stream. +func (p *payloadV1) Read(b []byte) (n int, err error) { + panic("not implemented") +} From 25440fd707bf059408da3d9a10ee992e94e47272 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Fri, 5 Sep 2025 15:24:39 -0400 Subject: [PATCH 02/75] move trace chunk to payload.go --- ddtrace/tracer/payload.go | 26 ++++++++++++++++++++++++++ ddtrace/tracer/payload_v1.go | 26 -------------------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/ddtrace/tracer/payload.go b/ddtrace/tracer/payload.go index 5730d37315..5214d8f447 100644 --- a/ddtrace/tracer/payload.go +++ b/ddtrace/tracer/payload.go @@ -150,3 +150,29 @@ func (sp *safePayload) protocol() float64 { // Protocol is immutable after creation - no lock needed return sp.p.protocol() } + +// traceChunk represents a list of spans with the same trace ID, +// i.e. a chunk of a trace +type traceChunk struct { + // the sampling priority of the trace + priority int32 + + // the optional string origin ("lambda", "rum", etc.) of the trace chunk + origin uint32 + + // a collection of key to value pairs common in all `spans` + attributes map[uint32]anyValue + + // a list of spans in this chunk + spans []Span + + // whether the trace only contains analyzed spans + // (not required by tracers and set by the agent) + droppedTrace bool + + // the ID of the trace to which all spans in this chunk belong + traceID uint8 + + // the optional string decision maker (previously span tag _dd.p.dm) + decisionMaker uint32 +} diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 38657baf5e..257b7c8d48 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -101,32 +101,6 @@ func newPayloadV1(protocol float64) *payloadV1 { } } -// traceChunk represents a list of spans with the same trace ID, -// i.e. a chunk of a trace -type traceChunk struct { - // the sampling priority of the trace - priority int32 - - // the optional string origin ("lambda", "rum", etc.) of the trace chunk - origin uint32 - - // a collection of key to value pairs common in all `spans` - attributes map[uint32]anyValue - - // a list of spans in this chunk - spans []Span - - // whether the trace only contains analyzed spans - // (not required by tracers and set by the agent) - droppedTrace bool - - // the ID of the trace to which all spans in this chunk belong - traceID uint8 - - // the optional string decision maker (previously span tag _dd.p.dm) - decisionMaker uint32 -} - func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { panic("not implemented") } From d0635e427d6434ded050c29e4d2e2c79a7934c03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dario=20Casta=C3=B1=C3=A9?= Date: Mon, 1 Sep 2025 14:36:14 +0200 Subject: [PATCH 03/75] feat(tracer): implement msgp serialization for payloadV1 and spanListV1 - Added msgp struct tags to fields in payloadV1 and traceChunk for serialization. - Introduced EncodeMsg methods for payloadV1 and spanListV1 to support msgp encoding. - Updated field types in traceChunk to enhance compatibility with msgp serialization. --- ddtrace/tracer/payload.go | 14 +++++------ ddtrace/tracer/payload_v1.go | 45 ++++++++++++++++++++++-------------- 2 files changed, 35 insertions(+), 24 deletions(-) diff --git a/ddtrace/tracer/payload.go b/ddtrace/tracer/payload.go index 5214d8f447..46088ab9d1 100644 --- a/ddtrace/tracer/payload.go +++ b/ddtrace/tracer/payload.go @@ -155,24 +155,24 @@ func (sp *safePayload) protocol() float64 { // i.e. a chunk of a trace type traceChunk struct { // the sampling priority of the trace - priority int32 + priority int32 `msg:"priority"` // the optional string origin ("lambda", "rum", etc.) of the trace chunk - origin uint32 + origin uint32 `msg:"origin,omitempty"` // a collection of key to value pairs common in all `spans` - attributes map[uint32]anyValue + attributes map[uint32]anyValue `msg:"attributes,omitempty"` // a list of spans in this chunk - spans []Span + spans spanListV1 `msg:"spans,omitempty"` // whether the trace only contains analyzed spans // (not required by tracers and set by the agent) - droppedTrace bool + droppedTrace bool `msg:"droppedTrace"` // the ID of the trace to which all spans in this chunk belong - traceID uint8 + traceID []byte `msg:"traceID"` // the optional string decision maker (previously span tag _dd.p.dm) - decisionMaker uint32 + decisionMaker uint32 `msg:"decisionMaker,omitempty"` } diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 257b7c8d48..e470eb5da6 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -5,6 +5,8 @@ package tracer +import "github.com/tinylib/msgp/msgp" + // payloadV1 is a new version of a msgp payload that can be sent to the agent. // Be aware that payloadV1 follows the same rules and constraints as payloadV04. That is: // @@ -20,41 +22,40 @@ package tracer // Close the request body before attempting to re-use it again! type payloadV1 struct { // array of strings referenced in this tracer payload, its chunks and spans - strings []string + strings []string `msgp:"strings"` // the string ID of the container where the tracer is running - containerID uint32 + containerID uint32 `msgp:"containerID"` // the string language name of the tracer - languageName uint32 + languageName uint32 `msgp:"languageName"` // the string language version of the tracer - languageVersion uint32 + languageVersion uint32 `msgp:"languageVersion"` // the string version of the tracer - tracerVersion uint32 + tracerVersion uint32 `msgp:"tracerVersion"` // the V4 string UUID representation of a tracer session - runtimeID uint32 + runtimeID uint32 `msgp:"runtimeID"` // the optional `env` string tag that set with the tracer - env uint32 + env uint32 `msgp:"env,omitempty"` // the optional string hostname of where the tracer is running - hostname uint32 + hostname uint32 `msgp:"hostname,omitempty"` // the optional string `version` tag for the application set in the tracer - appVersion uint32 + appVersion uint32 `msgp:"appVersion,omitempty"` // a collection of key to value pairs common in all `chunks` - attributes map[uint32]anyValue + attributes map[uint32]anyValue `msgp:"attributes,omitempty"` // a list of trace `chunks` - chunks []traceChunk + chunks []traceChunk `msgp:"chunks,omitempty"` - // fields needed to implement unsafePayload interface + // protocolVersion specifies the trace protocol to use. protocolVersion float64 - itemsCount uint32 } // AnyValue is a representation of the `any` value. It can take the following types: @@ -71,6 +72,13 @@ type anyValue struct { value interface{} } +// EncodeMsg implements msgp.Encodable. +func (a *anyValue) EncodeMsg(*msgp.Writer) error { + panic("unimplemented") +} + +var _ msgp.Encodable = (*anyValue)(nil) + const ( StringValueType = iota + 1 // string or uint BoolValueType // boolean @@ -92,17 +100,20 @@ type keyValue struct { type keyValueList = []keyValue // newPayloadV1 returns a ready to use payloadV1. -func newPayloadV1(protocol float64) *payloadV1 { +func newPayloadV1() *payloadV1 { return &payloadV1{ - protocolVersion: protocol, + protocolVersion: traceProtocolV1, strings: make([]string, 0), attributes: make(map[uint32]anyValue), chunks: make([]traceChunk, 0), } } -func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { - panic("not implemented") +// push pushes a new item into the stream. +func (p *payloadV1) push(t []*Span) error { + // We need to hydrate the payload with everything we get from the spans. + // Conceptually, our `t []*Span` corresponds to one `traceChunk`. + return nil } func (p *payloadV1) grow(n int) { From 478ef228230ef4fa56e2b90ffa9f8628ff88a0dd Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Fri, 5 Sep 2025 15:41:05 -0400 Subject: [PATCH 04/75] added missing spanlistv1 type --- ddtrace/tracer/payload.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ddtrace/tracer/payload.go b/ddtrace/tracer/payload.go index 46088ab9d1..f8e5285be1 100644 --- a/ddtrace/tracer/payload.go +++ b/ddtrace/tracer/payload.go @@ -63,6 +63,8 @@ const ( msgpackArray32 byte = 0xdd // up to 2^32-1 items, followed by size in 4 bytes ) +type spanListV1 []*Span + // safePayload provides a thread-safe wrapper around payload. type safePayload struct { mu sync.RWMutex From 230633af28da24d46b786be7fd820142fc2a7c14 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Fri, 5 Sep 2025 15:42:22 -0400 Subject: [PATCH 05/75] finish cherry picking dario's PR --- ddtrace/tracer/payload.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/ddtrace/tracer/payload.go b/ddtrace/tracer/payload.go index f8e5285be1..2f3f5f0995 100644 --- a/ddtrace/tracer/payload.go +++ b/ddtrace/tracer/payload.go @@ -8,6 +8,8 @@ package tracer import ( "io" "sync" + + "github.com/tinylib/msgp/msgp" ) // payloadStats contains the statistics of a payload. @@ -65,6 +67,17 @@ const ( type spanListV1 []*Span +// EncodeMsg implements msgp.Encodable. +func (s *spanListV1) EncodeMsg(*msgp.Writer) error { + // From here, encoding goes full manual. + // Span, SpanLink, and SpanEvent structs are different for v0.4 and v1.0. + // For v1 we need to manually encode the spans, span links, and span events + // if we don't want to do extra allocations. + panic("unimplemented") +} + +var _ msgp.Encodable = (*spanListV1)(nil) + // safePayload provides a thread-safe wrapper around payload. type safePayload struct { mu sync.RWMutex From c0f2493092e2a24ae5ef8ddb6b5193fc7fe84fcc Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Fri, 5 Sep 2025 15:58:39 -0400 Subject: [PATCH 06/75] fix newPayload and spanlist type --- ddtrace/tracer/payload.go | 12 +++++++----- ddtrace/tracer/payload_v1.go | 4 ++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/ddtrace/tracer/payload.go b/ddtrace/tracer/payload.go index 2f3f5f0995..1e3d56369e 100644 --- a/ddtrace/tracer/payload.go +++ b/ddtrace/tracer/payload.go @@ -48,11 +48,13 @@ type payload interface { payloadReader } -// newpayload returns a ready to use unsafe payload. +// newPayload returns a ready to use payload. func newPayload(protocol float64) payload { - // TODO(hannahkm): add support for v1 protocol - // if protocol == traceProtocolV1 { - // } + if protocol == traceProtocolV1 { + return &safePayload{ + p: newPayloadV1(), + } + } return &safePayload{ p: newPayloadV04(), } @@ -65,7 +67,7 @@ const ( msgpackArray32 byte = 0xdd // up to 2^32-1 items, followed by size in 4 bytes ) -type spanListV1 []*Span +type spanListV1 = spanList // EncodeMsg implements msgp.Encodable. func (s *spanListV1) EncodeMsg(*msgp.Writer) error { diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index e470eb5da6..b9624e459b 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -110,10 +110,10 @@ func newPayloadV1() *payloadV1 { } // push pushes a new item into the stream. -func (p *payloadV1) push(t []*Span) error { +func (p *payloadV1) push(t spanListV1) (stats payloadStats, err error) { // We need to hydrate the payload with everything we get from the spans. // Conceptually, our `t []*Span` corresponds to one `traceChunk`. - return nil + return payloadStats{}, nil } func (p *payloadV1) grow(n int) { From 1e941acb134cdbad3d343bb84f464fbb2b41f514 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Thu, 11 Sep 2025 17:55:03 -0400 Subject: [PATCH 07/75] wip: encode anyvalue and keyvalue --- ddtrace/tracer/payload_v1.go | 128 +++++++++++++++++++++++++++++++---- 1 file changed, 116 insertions(+), 12 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index b9624e459b..fff2a627ca 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -5,7 +5,13 @@ package tracer -import "github.com/tinylib/msgp/msgp" +import ( + "bytes" + "fmt" + "sync" + + "github.com/tinylib/msgp/msgp" +) // payloadV1 is a new version of a msgp payload that can be sent to the agent. // Be aware that payloadV1 follows the same rules and constraints as payloadV04. That is: @@ -22,7 +28,9 @@ import "github.com/tinylib/msgp/msgp" // Close the request body before attempting to re-use it again! type payloadV1 struct { // array of strings referenced in this tracer payload, its chunks and spans - strings []string `msgp:"strings"` + // stringTable holds references from a string value to an index. + // the 0th position in the stringTable should always be the empty string. + strings stringTable `msgp:"strings"` // the string ID of the container where the tracer is running containerID uint32 `msgp:"containerID"` @@ -56,6 +64,19 @@ type payloadV1 struct { // protocolVersion specifies the trace protocol to use. protocolVersion float64 + + // buf holds the sequence of msgpack-encoded items. + buf bytes.Buffer + + // reader is used for reading the contents of buf. + reader *bytes.Reader +} + +type stringTable struct { + m sync.Mutex + strings []string // list of strings + indices map[string]uint32 // map strings to their indices + nextIndex uint32 // last index of the stringTable } // AnyValue is a representation of the `any` value. It can take the following types: @@ -72,11 +93,6 @@ type anyValue struct { value interface{} } -// EncodeMsg implements msgp.Encodable. -func (a *anyValue) EncodeMsg(*msgp.Writer) error { - panic("unimplemented") -} - var _ msgp.Encodable = (*anyValue)(nil) const ( @@ -89,7 +105,7 @@ const ( keyValueListType // []keyValue ) -type arrayValue = []anyValue +type arrayValue []anyValue // keyValue is made up of the key and an AnyValue (the type of the value and the value itself) type keyValue struct { @@ -97,15 +113,21 @@ type keyValue struct { value anyValue } -type keyValueList = []keyValue +type keyValueList []keyValue + +var _ msgp.Encodable = (*keyValue)(nil) // newPayloadV1 returns a ready to use payloadV1. func newPayloadV1() *payloadV1 { return &payloadV1{ protocolVersion: traceProtocolV1, - strings: make([]string, 0), attributes: make(map[uint32]anyValue), chunks: make([]traceChunk, 0), + strings: stringTable{ + strings: []string{""}, + indices: map[string]uint32{"": 0}, + nextIndex: 1, + }, } } @@ -113,7 +135,14 @@ func newPayloadV1() *payloadV1 { func (p *payloadV1) push(t spanListV1) (stats payloadStats, err error) { // We need to hydrate the payload with everything we get from the spans. // Conceptually, our `t []*Span` corresponds to one `traceChunk`. - return payloadStats{}, nil + p.chunks = append(p.chunks, traceChunk{ + spans: t, + }) + if err := msgp.Encode(&p.buf, t); err != nil { // TODO(hannahkm): this needs to call (spanListV1).EncodeMsg + return payloadStats{}, err + } + p.recordItem() + return p.stats(), nil } func (p *payloadV1) grow(n int) { @@ -129,6 +158,7 @@ func (p *payloadV1) clear() { } func (p *payloadV1) recordItem() { + // atomic.AddUint32(&p.count, 1) panic("not implemented") } @@ -145,7 +175,7 @@ func (p *payloadV1) itemCount() int { } func (p *payloadV1) protocol() float64 { - panic("not implemented") + return p.protocolVersion } // Close implements io.Closer @@ -162,3 +192,77 @@ func (p *payloadV1) Write(data []byte) (n int, err error) { func (p *payloadV1) Read(b []byte) (n int, err error) { panic("not implemented") } + +// Encode the anyValue +// EncodeMsg implements msgp.Encodable. +func (a *anyValue) EncodeMsg(e *msgp.Writer) error { + switch a.valueType { + case StringValueType: + e.WriteInt32(StringValueType) + return encodeString(a.value.(string), e) + case BoolValueType: + e.WriteInt32(BoolValueType) + return e.WriteBool(a.value.(bool)) + case FloatValueType: + e.WriteInt32(FloatValueType) + return e.WriteFloat64(a.value.(float64)) + case IntValueType: + e.WriteInt32(IntValueType) + return e.WriteUint64(a.value.(uint64)) + case BytesValueType: + e.WriteInt32(BytesValueType) + return e.WriteBytes(a.value.([]byte)) + default: + return fmt.Errorf("invalid value type: %d", a.valueType) + } +} + +// EncodeMsg implements msgp.Encodable. +func (av arrayValue) EncodeMsg(e *msgp.Writer) error { + err := e.WriteArrayHeader(uint32(len(av))) + if err != nil { + return err + } + for _, value := range av { + if err := value.EncodeMsg(e); err != nil { + return err + } + } + return nil +} + +// EncodeMsg implements msgp.Encodable. +func (k keyValue) EncodeMsg(e *msgp.Writer) error { + err := e.WriteUint32(k.key) + if err != nil { + return err + } + err = k.value.EncodeMsg(e) + if err != nil { + return err + } + return nil +} + +func encodeKeyValueList(kv keyValueList, e *msgp.Writer) error { + err := e.WriteMapHeader(uint32(len(kv))) + if err != nil { + return err + } + for _, k := range kv { + if err := k.value.EncodeMsg(e); err != nil { + return err + } + } + return nil +} + +// When writing a string: +// - use its index in the string table if it exists +// - otherwise, write the string into the message, then add the string at the next index +// When reading a string, check that it is a uint and then: +// - if true, check read up the index position and return that position +// - else, add it to thenext index position and return that position +func encodeString(s string, e *msgp.Writer) error { + panic("not implemented") +} From dc00fd3d5e164bfe9b9006f1065ea85272d27203 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Thu, 11 Sep 2025 18:01:53 -0400 Subject: [PATCH 08/75] move spanLinkV1 into payload_v1 --- ddtrace/tracer/payload.go | 15 -------------- ddtrace/tracer/payload_v1.go | 39 ++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/ddtrace/tracer/payload.go b/ddtrace/tracer/payload.go index 1e3d56369e..998fc13b6c 100644 --- a/ddtrace/tracer/payload.go +++ b/ddtrace/tracer/payload.go @@ -8,8 +8,6 @@ package tracer import ( "io" "sync" - - "github.com/tinylib/msgp/msgp" ) // payloadStats contains the statistics of a payload. @@ -67,19 +65,6 @@ const ( msgpackArray32 byte = 0xdd // up to 2^32-1 items, followed by size in 4 bytes ) -type spanListV1 = spanList - -// EncodeMsg implements msgp.Encodable. -func (s *spanListV1) EncodeMsg(*msgp.Writer) error { - // From here, encoding goes full manual. - // Span, SpanLink, and SpanEvent structs are different for v0.4 and v1.0. - // For v1 we need to manually encode the spans, span links, and span events - // if we don't want to do extra allocations. - panic("unimplemented") -} - -var _ msgp.Encodable = (*spanListV1)(nil) - // safePayload provides a thread-safe wrapper around payload. type safePayload struct { mu sync.RWMutex diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index fff2a627ca..c05731d20d 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -193,6 +193,10 @@ func (p *payloadV1) Read(b []byte) (n int, err error) { panic("not implemented") } +type spanListV1 spanList + +var _ msgp.Encodable = (*spanListV1)(nil) + // Encode the anyValue // EncodeMsg implements msgp.Encodable. func (a *anyValue) EncodeMsg(e *msgp.Writer) error { @@ -257,6 +261,41 @@ func encodeKeyValueList(kv keyValueList, e *msgp.Writer) error { return nil } +// EncodeMsg writes the contents of the TraceChunk into `p.buf` +// Span, SpanLink, and SpanEvent structs are different for v0.4 and v1.0. +// For v1 we need to manually encode the spans, span links, and span events +// if we don't want to do extra allocations. +// EncodeMsg implements msgp.Encodable. +func (s spanListV1) EncodeMsg(e *msgp.Writer) error { + err := e.WriteArrayHeader(uint32(len(s))) + if err != nil { + return msgp.WrapError(err) + } + + e.WriteInt32(4) + for _, span := range s { + if span == nil { + err := e.WriteNil() + if err != nil { + return err + } + } else { + err := encodeSpan(span, e) + if err != nil { + return msgp.WrapError(err, span) + } + } + } + + return nil +} + +// Custom encoding for spans under the v1 trace protocol. +func encodeSpan(s *Span, e *msgp.Writer) error { + panic("not implemented") +} + +// encodeString and decodeString handles encoding a string to the payload's string table. // When writing a string: // - use its index in the string table if it exists // - otherwise, write the string into the message, then add the string at the next index From 98d30ceef9a3c90f0e7aae73be27ff9eb84fcdc2 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Fri, 12 Sep 2025 17:21:42 -0400 Subject: [PATCH 09/75] wip: encode spans --- ddtrace/tracer/payload_v1.go | 115 +++++++++++++++++++++++++++++++++-- 1 file changed, 109 insertions(+), 6 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index c05731d20d..9991fa1b6a 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -131,6 +131,13 @@ func newPayloadV1() *payloadV1 { } } +var _ msgp.Encodable = (*payloadV1)(nil) + +// EncodeMsg implements msgp.Encodable. +func (p *payloadV1) EncodeMsg(e *msgp.Writer) error { + panic("not implemented") +} + // push pushes a new item into the stream. func (p *payloadV1) push(t spanListV1) (stats payloadStats, err error) { // We need to hydrate the payload with everything we get from the spans. @@ -203,7 +210,11 @@ func (a *anyValue) EncodeMsg(e *msgp.Writer) error { switch a.valueType { case StringValueType: e.WriteInt32(StringValueType) - return encodeString(a.value.(string), e) + v, err := encodeString(a.value.(string)) + if err != nil { + return err + } + return e.WriteUint32(v) case BoolValueType: e.WriteInt32(BoolValueType) return e.WriteBool(a.value.(bool)) @@ -216,6 +227,9 @@ func (a *anyValue) EncodeMsg(e *msgp.Writer) error { case BytesValueType: e.WriteInt32(BytesValueType) return e.WriteBytes(a.value.([]byte)) + case ArrayValueType: + e.WriteInt32(ArrayValueType) + return a.value.(arrayValue).EncodeMsg(e) default: return fmt.Errorf("invalid value type: %d", a.valueType) } @@ -253,8 +267,13 @@ func encodeKeyValueList(kv keyValueList, e *msgp.Writer) error { if err != nil { return err } - for _, k := range kv { - if err := k.value.EncodeMsg(e); err != nil { + for i, k := range kv { + err := e.WriteUint32(uint32(i)) + if err != nil { + return err + } + err = k.EncodeMsg(e) + if err != nil { return err } } @@ -291,17 +310,101 @@ func (s spanListV1) EncodeMsg(e *msgp.Writer) error { } // Custom encoding for spans under the v1 trace protocol. +// The encoding of attributes is the combination of the meta, metrics, and metaStruct fields of the v0.4 protocol. func encodeSpan(s *Span, e *msgp.Writer) error { - panic("not implemented") + kv := keyValueList{ + {key: 1, value: anyValue{valueType: StringValueType, value: s.service}}, // service + {key: 2, value: anyValue{valueType: StringValueType, value: s.name}}, // name + {key: 3, value: anyValue{valueType: StringValueType, value: s.resource}}, // resource + {key: 4, value: anyValue{valueType: IntValueType, value: s.spanID}}, // spanID + {key: 5, value: anyValue{valueType: IntValueType, value: s.parentID}}, // parentID + {key: 6, value: anyValue{valueType: IntValueType, value: s.start}}, // start + {key: 7, value: anyValue{valueType: IntValueType, value: s.duration}}, // duration + {key: 8, value: anyValue{valueType: BoolValueType, value: s.error}}, // error + {key: 10, value: anyValue{valueType: StringValueType, value: s.spanType}}, // type + {key: 11, value: anyValue{valueType: keyValueListType, value: s.spanLinks}}, // SpanLink + {key: 12, value: anyValue{valueType: keyValueListType, value: s.spanEvents}}, // SpanEvent + {key: 15, value: anyValue{valueType: StringValueType, value: s.integration}}, // component + } + + // encode meta attributes + attr := keyValueList{} + for k, v := range s.meta { + idx, err := encodeString(k) + if err != nil { + // print something here + } + attr = append(attr, keyValue{key: idx, value: anyValue{valueType: StringValueType, value: v}}) + } + + // encode metric attributes + for k, v := range s.metrics { + idx, err := encodeString(k) + if err != nil { + // print something here + } + attr = append(attr, keyValue{key: idx, value: anyValue{valueType: FloatValueType, value: v}}) + } + + // encode metaStruct attributes + for k, v := range s.metaStruct { + idx, err := encodeString(k) + if err != nil { + // print something here + } + attr = append(attr, keyValue{key: idx, value: anyValue{valueType: getAnyValueType(v), value: v}}) + } + + kv = append(kv, keyValue{key: 9, value: anyValue{valueType: ArrayValueType, value: attr}}) // attributes + + env, ok := s.meta["env"] + if ok { + kv = append(kv, keyValue{key: 13, value: anyValue{valueType: StringValueType, value: env}}) // env + } + version, ok := s.meta["version"] + if ok { + kv = append(kv, keyValue{key: 14, value: anyValue{valueType: StringValueType, value: version}}) // version + } + + return encodeKeyValueList(kv, e) } // encodeString and decodeString handles encoding a string to the payload's string table. // When writing a string: // - use its index in the string table if it exists // - otherwise, write the string into the message, then add the string at the next index +// Returns the index of the string in the string table, and an error if there is one +func encodeString(s string) (uint32, error) { + panic("not implemented") +} + // When reading a string, check that it is a uint and then: // - if true, check read up the index position and return that position -// - else, add it to thenext index position and return that position -func encodeString(s string, e *msgp.Writer) error { +// - else, add it to the next index position and return that position +func decodeString(i uint32, e *msgp.Writer) (string, error) { panic("not implemented") } + +func encodeSpanLinks(sl []SpanLink, e *msgp.Writer) error { + panic("not implemented") +} + +func encodeSpanEvents(se []spanEvent, e *msgp.Writer) error { + panic("not implemented") +} + +func getAnyValueType(v any) int { + switch v.(type) { + case string: + return StringValueType + case bool: + return BoolValueType + case float64: + return FloatValueType + case float32: + return FloatValueType + case []byte: + return BytesValueType + } + return IntValueType +} From 0ad7dd12194223abc6b96c1d048b7e44adbadfd7 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Mon, 15 Sep 2025 14:36:49 -0400 Subject: [PATCH 10/75] wip: span event and span link --- ddtrace/tracer/payload_v1.go | 90 ++++++++++++++++++++++++++++++++---- 1 file changed, 81 insertions(+), 9 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 9991fa1b6a..e7f92331c9 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -96,13 +96,13 @@ type anyValue struct { var _ msgp.Encodable = (*anyValue)(nil) const ( - StringValueType = iota + 1 // string or uint - BoolValueType // boolean - FloatValueType // float64 - IntValueType // uint64 - BytesValueType // []uint8 - ArrayValueType // []AnyValue - keyValueListType // []keyValue + StringValueType = iota + 1 // string or uint -- 1 + BoolValueType // boolean -- 2 + FloatValueType // float64 -- 3 + IntValueType // uint64 -- 4 + BytesValueType // []uint8 -- 5 + ArrayValueType // []AnyValue -- 6 + keyValueListType // []keyValue -- 7 ) type arrayValue []anyValue @@ -385,12 +385,84 @@ func decodeString(i uint32, e *msgp.Writer) (string, error) { panic("not implemented") } +// encodeSpanLinks encodes the span links into a msgp.Writer +// Span links are represented as an array of fixmaps (keyValueList) func encodeSpanLinks(sl []SpanLink, e *msgp.Writer) error { - panic("not implemented") + // write the number of span links + err := e.WriteArrayHeader(uint32(len(sl))) + if err != nil { + return err + } + + // represent each span link as a fixmap (keyValueList) and add it to an array + kv := arrayValue{} + for _, s := range sl { + slKeyValues := keyValueList{ + {key: 1, value: anyValue{valueType: IntValueType, value: s.TraceID}}, // traceID + {key: 2, value: anyValue{valueType: IntValueType, value: s.SpanID}}, // spanID + {key: 4, value: anyValue{valueType: StringValueType, value: s.Tracestate}}, // tracestate + {key: 5, value: anyValue{valueType: IntValueType, value: s.Flags}}, // flags + } + + attr := keyValueList{} + // attributes + for k, v := range s.Attributes { + idx, err := encodeString(k) + if err != nil { + return err + } + attr = append(attr, keyValue{key: idx, value: anyValue{valueType: getAnyValueType(v), value: v}}) + } + slKeyValues = append(slKeyValues, keyValue{key: 3, value: anyValue{valueType: ArrayValueType, value: attr}}) // attributes + kv = append(kv, anyValue{valueType: keyValueListType, value: slKeyValues}) + } + + for _, v := range kv { + err := v.EncodeMsg(e) + if err != nil { + return err + } + } + return nil } +// encodeSpanEvents encodes the span events into a msgp.Writer +// Span events are represented as an array of fixmaps (keyValueList) func encodeSpanEvents(se []spanEvent, e *msgp.Writer) error { - panic("not implemented") + // write the number of span events + err := e.WriteArrayHeader(uint32(len(se))) + if err != nil { + return err + } + + // represent each span event as a fixmap (keyValueList) and add it to an array + kv := arrayValue{} + for _, s := range se { + slKeyValues := keyValueList{ + {key: 1, value: anyValue{valueType: IntValueType, value: s.TimeUnixNano}}, // time + {key: 2, value: anyValue{valueType: StringValueType, value: s.Name}}, // name + } + + attr := keyValueList{} + // attributes + for k, v := range s.Attributes { + idx, err := encodeString(k) + if err != nil { + return err + } + attr = append(attr, keyValue{key: idx, value: anyValue{valueType: getAnyValueType(v), value: v}}) + } + slKeyValues = append(slKeyValues, keyValue{key: 3, value: anyValue{valueType: ArrayValueType, value: attr}}) // attributes + kv = append(kv, anyValue{valueType: keyValueListType, value: slKeyValues}) + } + + for _, v := range kv { + err := v.EncodeMsg(e) + if err != nil { + return err + } + } + return nil } func getAnyValueType(v any) int { From 326a24442d1986325d16e71f5df053e93f76ab16 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Mon, 15 Sep 2025 15:05:24 -0400 Subject: [PATCH 11/75] wip: payload --- ddtrace/tracer/payload_v1.go | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index e7f92331c9..515b2d1580 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -116,6 +116,7 @@ type keyValue struct { type keyValueList []keyValue var _ msgp.Encodable = (*keyValue)(nil) +var _ msgp.Encodable = (keyValueList)(nil) // newPayloadV1 returns a ready to use payloadV1. func newPayloadV1() *payloadV1 { @@ -135,7 +136,19 @@ var _ msgp.Encodable = (*payloadV1)(nil) // EncodeMsg implements msgp.Encodable. func (p *payloadV1) EncodeMsg(e *msgp.Writer) error { - panic("not implemented") + kv := keyValueList{ + {key: 2, value: anyValue{valueType: IntValueType, value: p.containerID}}, // containerID + {key: 3, value: anyValue{valueType: IntValueType, value: p.languageName}}, // languageName + {key: 4, value: anyValue{valueType: IntValueType, value: p.languageVersion}}, // languageVersion + {key: 5, value: anyValue{valueType: IntValueType, value: p.tracerVersion}}, // tracerVersion + {key: 6, value: anyValue{valueType: IntValueType, value: p.runtimeID}}, // runtimeID + {key: 7, value: anyValue{valueType: StringValueType, value: p.env}}, // env + {key: 8, value: anyValue{valueType: StringValueType, value: p.hostname}}, // hostname + {key: 9, value: anyValue{valueType: StringValueType, value: p.appVersion}}, // appVersion + {key: 10, value: anyValue{valueType: keyValueListType, value: p.attributes}}, // attributes + {key: 11, value: anyValue{valueType: keyValueListType, value: p.chunks}}, // chunks + } + return kv.EncodeMsg(e) } // push pushes a new item into the stream. @@ -230,6 +243,9 @@ func (a *anyValue) EncodeMsg(e *msgp.Writer) error { case ArrayValueType: e.WriteInt32(ArrayValueType) return a.value.(arrayValue).EncodeMsg(e) + case keyValueListType: + e.WriteInt32(keyValueListType) + return a.value.(keyValueList).EncodeMsg(e) default: return fmt.Errorf("invalid value type: %d", a.valueType) } @@ -262,7 +278,7 @@ func (k keyValue) EncodeMsg(e *msgp.Writer) error { return nil } -func encodeKeyValueList(kv keyValueList, e *msgp.Writer) error { +func (kv keyValueList) EncodeMsg(e *msgp.Writer) error { err := e.WriteMapHeader(uint32(len(kv))) if err != nil { return err @@ -366,7 +382,7 @@ func encodeSpan(s *Span, e *msgp.Writer) error { kv = append(kv, keyValue{key: 14, value: anyValue{valueType: StringValueType, value: version}}) // version } - return encodeKeyValueList(kv, e) + return kv.EncodeMsg(e) } // encodeString and decodeString handles encoding a string to the payload's string table. From ea80c7a47e18841dacbadd3b12933043e4b73fab Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Tue, 16 Sep 2025 11:22:08 -0400 Subject: [PATCH 12/75] wip: encode traceChunk --- ddtrace/tracer/payload_v1.go | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 515b2d1580..08703c7fb9 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -117,6 +117,7 @@ type keyValueList []keyValue var _ msgp.Encodable = (*keyValue)(nil) var _ msgp.Encodable = (keyValueList)(nil) +var _ msgp.Encodable = (*traceChunk)(nil) // newPayloadV1 returns a ready to use payloadV1. func newPayloadV1() *payloadV1 { @@ -158,7 +159,7 @@ func (p *payloadV1) push(t spanListV1) (stats payloadStats, err error) { p.chunks = append(p.chunks, traceChunk{ spans: t, }) - if err := msgp.Encode(&p.buf, t); err != nil { // TODO(hannahkm): this needs to call (spanListV1).EncodeMsg + if err := t.EncodeMsg(&p.buf); err != nil { // TODO(hannahkm): this needs to call (spanListV1).EncodeMsg return payloadStats{}, err } p.recordItem() @@ -296,7 +297,26 @@ func (kv keyValueList) EncodeMsg(e *msgp.Writer) error { return nil } -// EncodeMsg writes the contents of the TraceChunk into `p.buf` +func (t *traceChunk) EncodeMsg(e *msgp.Writer) error { + kv := keyValueList{ + {key: 1, value: anyValue{valueType: IntValueType, value: t.priority}}, // priority + {key: 2, value: anyValue{valueType: StringValueType, value: t.origin}}, // origin + {key: 4, value: anyValue{valueType: keyValueListType, value: t.spans}}, // spans + {key: 5, value: anyValue{valueType: BoolValueType, value: t.droppedTrace}}, // droppedTrace + {key: 6, value: anyValue{valueType: BytesValueType, value: t.traceID}}, // traceID + {key: 7, value: anyValue{valueType: IntValueType, value: t.decisionMaker}}, // samplingMechanism + } + + attr := keyValueList{} + for k, v := range t.attributes { + attr = append(attr, keyValue{key: k, value: anyValue{valueType: getAnyValueType(v), value: v}}) + } + kv = append(kv, keyValue{key: 3, value: anyValue{valueType: ArrayValueType, value: attr}}) // attributes + + return kv.EncodeMsg(e) +} + +// EncodeMsg writes the contents of a list of spans into `p.buf` // Span, SpanLink, and SpanEvent structs are different for v0.4 and v1.0. // For v1 we need to manually encode the spans, span links, and span events // if we don't want to do extra allocations. From af26f1501b3d2aa5bc237cf2ea4b19ca8177adb3 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Wed, 17 Sep 2025 14:21:06 -0400 Subject: [PATCH 13/75] fix payload encodings --- ddtrace/tracer/payload_v1.go | 108 +++++++++++++++++++++++------------ 1 file changed, 71 insertions(+), 37 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 08703c7fb9..97106542b9 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -7,8 +7,10 @@ package tracer import ( "bytes" + "encoding/binary" "fmt" "sync" + "sync/atomic" "github.com/tinylib/msgp/msgp" ) @@ -65,6 +67,17 @@ type payloadV1 struct { // protocolVersion specifies the trace protocol to use. protocolVersion float64 + // header specifies the first few bytes in the msgpack stream + // indicating the type of array (fixarray, array16 or array32) + // and the number of items contained in the stream. + header []byte + + // off specifies the current read position on the header. + off int + + // count specifies the number of items in the stream. + count uint32 + // buf holds the sequence of msgpack-encoded items. buf bytes.Buffer @@ -133,33 +146,30 @@ func newPayloadV1() *payloadV1 { } } -var _ msgp.Encodable = (*payloadV1)(nil) - -// EncodeMsg implements msgp.Encodable. -func (p *payloadV1) EncodeMsg(e *msgp.Writer) error { - kv := keyValueList{ - {key: 2, value: anyValue{valueType: IntValueType, value: p.containerID}}, // containerID - {key: 3, value: anyValue{valueType: IntValueType, value: p.languageName}}, // languageName - {key: 4, value: anyValue{valueType: IntValueType, value: p.languageVersion}}, // languageVersion - {key: 5, value: anyValue{valueType: IntValueType, value: p.tracerVersion}}, // tracerVersion - {key: 6, value: anyValue{valueType: IntValueType, value: p.runtimeID}}, // runtimeID - {key: 7, value: anyValue{valueType: StringValueType, value: p.env}}, // env - {key: 8, value: anyValue{valueType: StringValueType, value: p.hostname}}, // hostname - {key: 9, value: anyValue{valueType: StringValueType, value: p.appVersion}}, // appVersion - {key: 10, value: anyValue{valueType: keyValueListType, value: p.attributes}}, // attributes - {key: 11, value: anyValue{valueType: keyValueListType, value: p.chunks}}, // chunks - } - return kv.EncodeMsg(e) -} - // push pushes a new item into the stream. func (p *payloadV1) push(t spanListV1) (stats payloadStats, err error) { // We need to hydrate the payload with everything we get from the spans. // Conceptually, our `t []*Span` corresponds to one `traceChunk`. + origin, priority := "", 0 + for _, span := range t { + if span == nil { + continue + } + if p, ok := span.Context().SamplingPriority(); ok { + origin = span.Context().origin + priority = p + break + } + } + p.chunks = append(p.chunks, traceChunk{ - spans: t, + priority: int32(priority), + origin: origin, + attributes: make(map[uint32]anyValue), + spans: t, + traceID: t[0].Context().traceID, }) - if err := t.EncodeMsg(&p.buf); err != nil { // TODO(hannahkm): this needs to call (spanListV1).EncodeMsg + if err := msgp.Encode(&p.buf, t); err != nil { return payloadStats{}, err } p.recordItem() @@ -167,46 +177,70 @@ func (p *payloadV1) push(t spanListV1) (stats payloadStats, err error) { } func (p *payloadV1) grow(n int) { - panic("not implemented") + p.buf.Grow(n) } func (p *payloadV1) reset() { - panic("not implemented") + p.updateHeader() + if p.reader != nil { + p.reader.Seek(0, 0) + } } func (p *payloadV1) clear() { - panic("not implemented") + p.buf = bytes.Buffer{} + p.reader = nil } func (p *payloadV1) recordItem() { - // atomic.AddUint32(&p.count, 1) - panic("not implemented") + atomic.AddUint32(&p.count, 1) + p.updateHeader() } func (p *payloadV1) stats() payloadStats { - panic("not implemented") + return payloadStats{ + size: p.size(), + itemCount: p.itemCount(), + } } func (p *payloadV1) size() int { - panic("not implemented") + return p.buf.Len() + len(p.header) - p.off } func (p *payloadV1) itemCount() int { - panic("not implemented") + return int(atomic.LoadUint32(&p.count)) } func (p *payloadV1) protocol() float64 { return p.protocolVersion } +func (p *payloadV1) updateHeader() { + n := uint64(atomic.LoadUint32(&p.count)) + switch { + case n <= 15: + p.header[7] = msgpackArrayFix + byte(n) + p.off = 7 + case n <= 1<<16-1: + binary.BigEndian.PutUint64(p.header, n) // writes 2 bytes + p.header[5] = msgpackArray16 + p.off = 5 + default: // n <= 1<<32-1 + binary.BigEndian.PutUint64(p.header, n) // writes 4 bytes + p.header[3] = msgpackArray32 + p.off = 3 + } +} + // Close implements io.Closer func (p *payloadV1) Close() error { - panic("not implemented") + return nil } // Write implements io.Writer. It writes data directly to the buffer. func (p *payloadV1) Write(data []byte) (n int, err error) { - panic("not implemented") + return p.buf.Write(data) } // Read implements io.Reader. It reads from the msgpack-encoded stream. @@ -299,12 +333,12 @@ func (kv keyValueList) EncodeMsg(e *msgp.Writer) error { func (t *traceChunk) EncodeMsg(e *msgp.Writer) error { kv := keyValueList{ - {key: 1, value: anyValue{valueType: IntValueType, value: t.priority}}, // priority - {key: 2, value: anyValue{valueType: StringValueType, value: t.origin}}, // origin - {key: 4, value: anyValue{valueType: keyValueListType, value: t.spans}}, // spans - {key: 5, value: anyValue{valueType: BoolValueType, value: t.droppedTrace}}, // droppedTrace - {key: 6, value: anyValue{valueType: BytesValueType, value: t.traceID}}, // traceID - {key: 7, value: anyValue{valueType: IntValueType, value: t.decisionMaker}}, // samplingMechanism + {key: 1, value: anyValue{valueType: IntValueType, value: t.priority}}, // priority + {key: 2, value: anyValue{valueType: StringValueType, value: t.origin}}, // origin + {key: 4, value: anyValue{valueType: keyValueListType, value: t.spans}}, // spans + {key: 5, value: anyValue{valueType: BoolValueType, value: t.droppedTrace}}, // droppedTrace + {key: 6, value: anyValue{valueType: BytesValueType, value: t.traceID}}, // traceID + {key: 7, value: anyValue{valueType: IntValueType, value: t.samplingMechanism}}, // samplingMechanism } attr := keyValueList{} From ded707db27288e539c9f1a32742f033220f25c82 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Wed, 17 Sep 2025 15:18:28 -0400 Subject: [PATCH 14/75] fix traceChunk field types --- ddtrace/tracer/payload.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ddtrace/tracer/payload.go b/ddtrace/tracer/payload.go index 998fc13b6c..7fd8c402b5 100644 --- a/ddtrace/tracer/payload.go +++ b/ddtrace/tracer/payload.go @@ -160,7 +160,7 @@ type traceChunk struct { priority int32 `msg:"priority"` // the optional string origin ("lambda", "rum", etc.) of the trace chunk - origin uint32 `msg:"origin,omitempty"` + origin string `msg:"origin,omitempty"` // a collection of key to value pairs common in all `spans` attributes map[uint32]anyValue `msg:"attributes,omitempty"` @@ -173,8 +173,8 @@ type traceChunk struct { droppedTrace bool `msg:"droppedTrace"` // the ID of the trace to which all spans in this chunk belong - traceID []byte `msg:"traceID"` + traceID [16]byte `msg:"traceID"` // the optional string decision maker (previously span tag _dd.p.dm) - decisionMaker uint32 `msg:"decisionMaker,omitempty"` + samplingMechanism string `msg:"samplingMechanism,omitempty"` } From 5124410174cce24fa206f120a157a5834c4aae08 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Thu, 18 Sep 2025 15:40:53 -0400 Subject: [PATCH 15/75] broken string table implementation --- ddtrace/tracer/payload_v1.go | 40 +++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 97106542b9..e1d386e8cd 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -92,6 +92,11 @@ type stringTable struct { nextIndex uint32 // last index of the stringTable } +type payloadWrapper struct { + *msgp.Writer + payload *payloadV1 +} + // AnyValue is a representation of the `any` value. It can take the following types: // - uint32 // - bool @@ -260,7 +265,7 @@ func (a *anyValue) EncodeMsg(e *msgp.Writer) error { e.WriteInt32(StringValueType) v, err := encodeString(a.value.(string)) if err != nil { - return err + return e.WriteString(a.value.(string)) } return e.WriteUint32(v) case BoolValueType: @@ -402,7 +407,7 @@ func encodeSpan(s *Span, e *msgp.Writer) error { for k, v := range s.meta { idx, err := encodeString(k) if err != nil { - // print something here + idx = k } attr = append(attr, keyValue{key: idx, value: anyValue{valueType: StringValueType, value: v}}) } @@ -411,7 +416,7 @@ func encodeSpan(s *Span, e *msgp.Writer) error { for k, v := range s.metrics { idx, err := encodeString(k) if err != nil { - // print something here + idx = k } attr = append(attr, keyValue{key: idx, value: anyValue{valueType: FloatValueType, value: v}}) } @@ -420,7 +425,7 @@ func encodeSpan(s *Span, e *msgp.Writer) error { for k, v := range s.metaStruct { idx, err := encodeString(k) if err != nil { - // print something here + idx = k } attr = append(attr, keyValue{key: idx, value: anyValue{valueType: getAnyValueType(v), value: v}}) } @@ -444,15 +449,22 @@ func encodeSpan(s *Span, e *msgp.Writer) error { // - use its index in the string table if it exists // - otherwise, write the string into the message, then add the string at the next index // Returns the index of the string in the string table, and an error if there is one -func encodeString(s string) (uint32, error) { - panic("not implemented") -} +func (p *payloadV1) encodeString(s string) (uint32, error) { + sTable := &p.strings + sTable.m.Lock() + defer sTable.m.Unlock() + idx, ok := sTable.indices[s] + // if the string already exists in the table, use its index + if ok { + return idx, nil + } -// When reading a string, check that it is a uint and then: -// - if true, check read up the index position and return that position -// - else, add it to the next index position and return that position -func decodeString(i uint32, e *msgp.Writer) (string, error) { - panic("not implemented") + // else, write the string into the table at the next index + // return an error to indicate that the string should be written to the msgp message + sTable.indices[s] = sTable.nextIndex + sTable.strings = append(sTable.strings, s) + sTable.nextIndex += 1 + return sTable.nextIndex, fmt.Errorf("string not found in table") } // encodeSpanLinks encodes the span links into a msgp.Writer @@ -479,7 +491,7 @@ func encodeSpanLinks(sl []SpanLink, e *msgp.Writer) error { for k, v := range s.Attributes { idx, err := encodeString(k) if err != nil { - return err + idx = k } attr = append(attr, keyValue{key: idx, value: anyValue{valueType: getAnyValueType(v), value: v}}) } @@ -518,7 +530,7 @@ func encodeSpanEvents(se []spanEvent, e *msgp.Writer) error { for k, v := range s.Attributes { idx, err := encodeString(k) if err != nil { - return err + idx = k } attr = append(attr, keyValue{key: idx, value: anyValue{valueType: getAnyValueType(v), value: v}}) } From bb1baaf1c04d6d57e9715e77fd77991b2635541b Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Mon, 22 Sep 2025 14:36:22 -0400 Subject: [PATCH 16/75] clean up some things --- ddtrace/tracer/payload_v1.go | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index e1d386e8cd..4981a3d9d9 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -9,7 +9,6 @@ import ( "bytes" "encoding/binary" "fmt" - "sync" "sync/atomic" "github.com/tinylib/msgp/msgp" @@ -86,17 +85,11 @@ type payloadV1 struct { } type stringTable struct { - m sync.Mutex strings []string // list of strings indices map[string]uint32 // map strings to their indices nextIndex uint32 // last index of the stringTable } -type payloadWrapper struct { - *msgp.Writer - payload *payloadV1 -} - // AnyValue is a representation of the `any` value. It can take the following types: // - uint32 // - bool @@ -111,8 +104,6 @@ type anyValue struct { value interface{} } -var _ msgp.Encodable = (*anyValue)(nil) - const ( StringValueType = iota + 1 // string or uint -- 1 BoolValueType // boolean -- 2 @@ -133,6 +124,10 @@ type keyValue struct { type keyValueList []keyValue +type spanListV1 spanList + +var _ msgp.Encodable = (*anyValue)(nil) +var _ msgp.Encodable = (*spanListV1)(nil) var _ msgp.Encodable = (*keyValue)(nil) var _ msgp.Encodable = (keyValueList)(nil) var _ msgp.Encodable = (*traceChunk)(nil) @@ -253,10 +248,6 @@ func (p *payloadV1) Read(b []byte) (n int, err error) { panic("not implemented") } -type spanListV1 spanList - -var _ msgp.Encodable = (*spanListV1)(nil) - // Encode the anyValue // EncodeMsg implements msgp.Encodable. func (a *anyValue) EncodeMsg(e *msgp.Writer) error { @@ -451,8 +442,6 @@ func encodeSpan(s *Span, e *msgp.Writer) error { // Returns the index of the string in the string table, and an error if there is one func (p *payloadV1) encodeString(s string) (uint32, error) { sTable := &p.strings - sTable.m.Lock() - defer sTable.m.Unlock() idx, ok := sTable.indices[s] // if the string already exists in the table, use its index if ok { From f071e05f2360d16845eb4f8e26f3fa7b676c3f23 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Tue, 23 Sep 2025 14:22:21 -0400 Subject: [PATCH 17/75] streamingKey type for stringTable (oh god) --- ddtrace/tracer/payload_v1.go | 170 +++++++++++++++++++---------------- 1 file changed, 91 insertions(+), 79 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 4981a3d9d9..894afb4fd0 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -116,9 +116,18 @@ const ( type arrayValue []anyValue +// keys in a keyValue can either be a string or a uint32 index +// isString is true when the key is a string value, and false when the key is a uint32 index +type streamingKey struct { + isString bool + stringValue string + idx uint32 +} + // keyValue is made up of the key and an AnyValue (the type of the value and the value itself) +// The key is either a uint32 index into the string table or a string value. type keyValue struct { - key uint32 + key streamingKey value anyValue } @@ -126,12 +135,6 @@ type keyValueList []keyValue type spanListV1 spanList -var _ msgp.Encodable = (*anyValue)(nil) -var _ msgp.Encodable = (*spanListV1)(nil) -var _ msgp.Encodable = (*keyValue)(nil) -var _ msgp.Encodable = (keyValueList)(nil) -var _ msgp.Encodable = (*traceChunk)(nil) - // newPayloadV1 returns a ready to use payloadV1. func newPayloadV1() *payloadV1 { return &payloadV1{ @@ -169,7 +172,12 @@ func (p *payloadV1) push(t spanListV1) (stats payloadStats, err error) { spans: t, traceID: t[0].Context().traceID, }) - if err := msgp.Encode(&p.buf, t); err != nil { + wr := msgp.NewWriter(&p.buf) + err = t.EncodeMsg(wr, p) + if err == nil { + err = wr.Flush() + } + if err != nil { return payloadStats{}, err } p.recordItem() @@ -249,16 +257,18 @@ func (p *payloadV1) Read(b []byte) (n int, err error) { } // Encode the anyValue -// EncodeMsg implements msgp.Encodable. -func (a *anyValue) EncodeMsg(e *msgp.Writer) error { +func (a *anyValue) EncodeMsg(e *msgp.Writer, p *payloadV1) error { switch a.valueType { case StringValueType: e.WriteInt32(StringValueType) - v, err := encodeString(a.value.(string)) + v, err := p.encodeString(a.value.(string)) if err != nil { - return e.WriteString(a.value.(string)) + return err } - return e.WriteUint32(v) + if v.isString { + return e.WriteString(v.stringValue) + } + return e.WriteUint32(v.idx) case BoolValueType: e.WriteInt32(BoolValueType) return e.WriteBool(a.value.(bool)) @@ -273,43 +283,46 @@ func (a *anyValue) EncodeMsg(e *msgp.Writer) error { return e.WriteBytes(a.value.([]byte)) case ArrayValueType: e.WriteInt32(ArrayValueType) - return a.value.(arrayValue).EncodeMsg(e) + return a.value.(arrayValue).EncodeMsg(e, p) case keyValueListType: e.WriteInt32(keyValueListType) - return a.value.(keyValueList).EncodeMsg(e) + return a.value.(keyValueList).EncodeMsg(e, p) default: return fmt.Errorf("invalid value type: %d", a.valueType) } } -// EncodeMsg implements msgp.Encodable. -func (av arrayValue) EncodeMsg(e *msgp.Writer) error { +func (av arrayValue) EncodeMsg(e *msgp.Writer, p *payloadV1) error { err := e.WriteArrayHeader(uint32(len(av))) if err != nil { return err } for _, value := range av { - if err := value.EncodeMsg(e); err != nil { + if err := value.EncodeMsg(e, p); err != nil { return err } } return nil } -// EncodeMsg implements msgp.Encodable. -func (k keyValue) EncodeMsg(e *msgp.Writer) error { - err := e.WriteUint32(k.key) +func (k keyValue) EncodeMsg(e *msgp.Writer, p *payloadV1) error { + var err error + if k.key.isString { + err = e.WriteString(k.key.stringValue) + } else { + err = e.WriteUint32(k.key.idx) + } if err != nil { return err } - err = k.value.EncodeMsg(e) + err = k.value.EncodeMsg(e, p) if err != nil { return err } return nil } -func (kv keyValueList) EncodeMsg(e *msgp.Writer) error { +func (kv keyValueList) EncodeMsg(e *msgp.Writer, p *payloadV1) error { err := e.WriteMapHeader(uint32(len(kv))) if err != nil { return err @@ -319,7 +332,7 @@ func (kv keyValueList) EncodeMsg(e *msgp.Writer) error { if err != nil { return err } - err = k.EncodeMsg(e) + err = k.EncodeMsg(e, p) if err != nil { return err } @@ -327,31 +340,30 @@ func (kv keyValueList) EncodeMsg(e *msgp.Writer) error { return nil } -func (t *traceChunk) EncodeMsg(e *msgp.Writer) error { +func (t *traceChunk) EncodeMsg(e *msgp.Writer, p *payloadV1) error { kv := keyValueList{ - {key: 1, value: anyValue{valueType: IntValueType, value: t.priority}}, // priority - {key: 2, value: anyValue{valueType: StringValueType, value: t.origin}}, // origin - {key: 4, value: anyValue{valueType: keyValueListType, value: t.spans}}, // spans - {key: 5, value: anyValue{valueType: BoolValueType, value: t.droppedTrace}}, // droppedTrace - {key: 6, value: anyValue{valueType: BytesValueType, value: t.traceID}}, // traceID - {key: 7, value: anyValue{valueType: IntValueType, value: t.samplingMechanism}}, // samplingMechanism + {key: streamingKey{isString: false, idx: 1}, value: anyValue{valueType: IntValueType, value: t.priority}}, // priority + {key: streamingKey{isString: false, idx: 2}, value: anyValue{valueType: StringValueType, value: t.origin}}, // origin + {key: streamingKey{isString: false, idx: 4}, value: anyValue{valueType: keyValueListType, value: t.spans}}, // spans + {key: streamingKey{isString: false, idx: 5}, value: anyValue{valueType: BoolValueType, value: t.droppedTrace}}, // droppedTrace + {key: streamingKey{isString: false, idx: 6}, value: anyValue{valueType: BytesValueType, value: t.traceID}}, // traceID + {key: streamingKey{isString: false, idx: 7}, value: anyValue{valueType: IntValueType, value: t.samplingMechanism}}, // samplingMechanism } attr := keyValueList{} for k, v := range t.attributes { - attr = append(attr, keyValue{key: k, value: anyValue{valueType: getAnyValueType(v), value: v}}) + attr = append(attr, keyValue{key: streamingKey{isString: false, idx: k}, value: anyValue{valueType: getAnyValueType(v), value: v}}) } - kv = append(kv, keyValue{key: 3, value: anyValue{valueType: ArrayValueType, value: attr}}) // attributes + kv = append(kv, keyValue{key: streamingKey{isString: false, idx: 3}, value: anyValue{valueType: ArrayValueType, value: attr}}) // attributes - return kv.EncodeMsg(e) + return kv.EncodeMsg(e, p) } // EncodeMsg writes the contents of a list of spans into `p.buf` // Span, SpanLink, and SpanEvent structs are different for v0.4 and v1.0. // For v1 we need to manually encode the spans, span links, and span events // if we don't want to do extra allocations. -// EncodeMsg implements msgp.Encodable. -func (s spanListV1) EncodeMsg(e *msgp.Writer) error { +func (s spanListV1) EncodeMsg(e *msgp.Writer, p *payloadV1) error { err := e.WriteArrayHeader(uint32(len(s))) if err != nil { return msgp.WrapError(err) @@ -365,7 +377,7 @@ func (s spanListV1) EncodeMsg(e *msgp.Writer) error { return err } } else { - err := encodeSpan(span, e) + err := encodeSpan(span, e, p) if err != nil { return msgp.WrapError(err, span) } @@ -377,62 +389,62 @@ func (s spanListV1) EncodeMsg(e *msgp.Writer) error { // Custom encoding for spans under the v1 trace protocol. // The encoding of attributes is the combination of the meta, metrics, and metaStruct fields of the v0.4 protocol. -func encodeSpan(s *Span, e *msgp.Writer) error { +func encodeSpan(s *Span, e *msgp.Writer, p *payloadV1) error { kv := keyValueList{ - {key: 1, value: anyValue{valueType: StringValueType, value: s.service}}, // service - {key: 2, value: anyValue{valueType: StringValueType, value: s.name}}, // name - {key: 3, value: anyValue{valueType: StringValueType, value: s.resource}}, // resource - {key: 4, value: anyValue{valueType: IntValueType, value: s.spanID}}, // spanID - {key: 5, value: anyValue{valueType: IntValueType, value: s.parentID}}, // parentID - {key: 6, value: anyValue{valueType: IntValueType, value: s.start}}, // start - {key: 7, value: anyValue{valueType: IntValueType, value: s.duration}}, // duration - {key: 8, value: anyValue{valueType: BoolValueType, value: s.error}}, // error - {key: 10, value: anyValue{valueType: StringValueType, value: s.spanType}}, // type - {key: 11, value: anyValue{valueType: keyValueListType, value: s.spanLinks}}, // SpanLink - {key: 12, value: anyValue{valueType: keyValueListType, value: s.spanEvents}}, // SpanEvent - {key: 15, value: anyValue{valueType: StringValueType, value: s.integration}}, // component + {key: streamingKey{isString: false, idx: 1}, value: anyValue{valueType: StringValueType, value: s.service}}, // service + {key: streamingKey{isString: false, idx: 2}, value: anyValue{valueType: StringValueType, value: s.name}}, // name + {key: streamingKey{isString: false, idx: 3}, value: anyValue{valueType: StringValueType, value: s.resource}}, // resource + {key: streamingKey{isString: false, idx: 4}, value: anyValue{valueType: IntValueType, value: s.spanID}}, // spanID + {key: streamingKey{isString: false, idx: 5}, value: anyValue{valueType: IntValueType, value: s.parentID}}, // parentID + {key: streamingKey{isString: false, idx: 6}, value: anyValue{valueType: IntValueType, value: s.start}}, // start + {key: streamingKey{isString: false, idx: 7}, value: anyValue{valueType: IntValueType, value: s.duration}}, // duration + {key: streamingKey{isString: false, idx: 8}, value: anyValue{valueType: BoolValueType, value: s.error}}, // error + {key: streamingKey{isString: false, idx: 10}, value: anyValue{valueType: StringValueType, value: s.spanType}}, // type + {key: streamingKey{isString: false, idx: 11}, value: anyValue{valueType: keyValueListType, value: s.spanLinks}}, // SpanLink + {key: streamingKey{isString: false, idx: 12}, value: anyValue{valueType: keyValueListType, value: s.spanEvents}}, // SpanEvent + {key: streamingKey{isString: false, idx: 15}, value: anyValue{valueType: StringValueType, value: s.integration}}, // component } // encode meta attributes attr := keyValueList{} for k, v := range s.meta { - idx, err := encodeString(k) + idx, err := p.encodeString(k) if err != nil { - idx = k + idx = streamingKey{isString: true, stringValue: k} } attr = append(attr, keyValue{key: idx, value: anyValue{valueType: StringValueType, value: v}}) } // encode metric attributes for k, v := range s.metrics { - idx, err := encodeString(k) + idx, err := p.encodeString(k) if err != nil { - idx = k + idx = streamingKey{isString: true, stringValue: k} } attr = append(attr, keyValue{key: idx, value: anyValue{valueType: FloatValueType, value: v}}) } // encode metaStruct attributes for k, v := range s.metaStruct { - idx, err := encodeString(k) + idx, err := p.encodeString(k) if err != nil { - idx = k + idx = streamingKey{isString: true, stringValue: k} } attr = append(attr, keyValue{key: idx, value: anyValue{valueType: getAnyValueType(v), value: v}}) } - kv = append(kv, keyValue{key: 9, value: anyValue{valueType: ArrayValueType, value: attr}}) // attributes + kv = append(kv, keyValue{key: streamingKey{isString: false, idx: 9}, value: anyValue{valueType: ArrayValueType, value: attr}}) // attributes env, ok := s.meta["env"] if ok { - kv = append(kv, keyValue{key: 13, value: anyValue{valueType: StringValueType, value: env}}) // env + kv = append(kv, keyValue{key: streamingKey{isString: false, idx: 13}, value: anyValue{valueType: StringValueType, value: env}}) // env } version, ok := s.meta["version"] if ok { - kv = append(kv, keyValue{key: 14, value: anyValue{valueType: StringValueType, value: version}}) // version + kv = append(kv, keyValue{key: streamingKey{isString: false, idx: 14}, value: anyValue{valueType: StringValueType, value: version}}) // version } - return kv.EncodeMsg(e) + return kv.EncodeMsg(e, p) } // encodeString and decodeString handles encoding a string to the payload's string table. @@ -440,12 +452,12 @@ func encodeSpan(s *Span, e *msgp.Writer) error { // - use its index in the string table if it exists // - otherwise, write the string into the message, then add the string at the next index // Returns the index of the string in the string table, and an error if there is one -func (p *payloadV1) encodeString(s string) (uint32, error) { +func (p *payloadV1) encodeString(s string) (streamingKey, error) { sTable := &p.strings idx, ok := sTable.indices[s] // if the string already exists in the table, use its index if ok { - return idx, nil + return streamingKey{isString: false, idx: idx}, nil } // else, write the string into the table at the next index @@ -453,12 +465,12 @@ func (p *payloadV1) encodeString(s string) (uint32, error) { sTable.indices[s] = sTable.nextIndex sTable.strings = append(sTable.strings, s) sTable.nextIndex += 1 - return sTable.nextIndex, fmt.Errorf("string not found in table") + return streamingKey{isString: true, stringValue: s}, nil } // encodeSpanLinks encodes the span links into a msgp.Writer // Span links are represented as an array of fixmaps (keyValueList) -func encodeSpanLinks(sl []SpanLink, e *msgp.Writer) error { +func encodeSpanLinks(sl []SpanLink, e *msgp.Writer, p *payloadV1) error { // write the number of span links err := e.WriteArrayHeader(uint32(len(sl))) if err != nil { @@ -469,27 +481,27 @@ func encodeSpanLinks(sl []SpanLink, e *msgp.Writer) error { kv := arrayValue{} for _, s := range sl { slKeyValues := keyValueList{ - {key: 1, value: anyValue{valueType: IntValueType, value: s.TraceID}}, // traceID - {key: 2, value: anyValue{valueType: IntValueType, value: s.SpanID}}, // spanID - {key: 4, value: anyValue{valueType: StringValueType, value: s.Tracestate}}, // tracestate - {key: 5, value: anyValue{valueType: IntValueType, value: s.Flags}}, // flags + {key: streamingKey{isString: false, idx: 1}, value: anyValue{valueType: IntValueType, value: s.TraceID}}, // traceID + {key: streamingKey{isString: false, idx: 2}, value: anyValue{valueType: IntValueType, value: s.SpanID}}, // spanID + {key: streamingKey{isString: false, idx: 4}, value: anyValue{valueType: StringValueType, value: s.Tracestate}}, // tracestate + {key: streamingKey{isString: false, idx: 5}, value: anyValue{valueType: IntValueType, value: s.Flags}}, // flags } attr := keyValueList{} // attributes for k, v := range s.Attributes { - idx, err := encodeString(k) + idx, err := p.encodeString(k) if err != nil { - idx = k + idx = streamingKey{isString: true, stringValue: k} } attr = append(attr, keyValue{key: idx, value: anyValue{valueType: getAnyValueType(v), value: v}}) } - slKeyValues = append(slKeyValues, keyValue{key: 3, value: anyValue{valueType: ArrayValueType, value: attr}}) // attributes + slKeyValues = append(slKeyValues, keyValue{key: streamingKey{isString: false, idx: 3}, value: anyValue{valueType: ArrayValueType, value: attr}}) // attributes kv = append(kv, anyValue{valueType: keyValueListType, value: slKeyValues}) } for _, v := range kv { - err := v.EncodeMsg(e) + err := v.EncodeMsg(e, p) if err != nil { return err } @@ -499,7 +511,7 @@ func encodeSpanLinks(sl []SpanLink, e *msgp.Writer) error { // encodeSpanEvents encodes the span events into a msgp.Writer // Span events are represented as an array of fixmaps (keyValueList) -func encodeSpanEvents(se []spanEvent, e *msgp.Writer) error { +func encodeSpanEvents(se []spanEvent, e *msgp.Writer, p *payloadV1) error { // write the number of span events err := e.WriteArrayHeader(uint32(len(se))) if err != nil { @@ -510,25 +522,25 @@ func encodeSpanEvents(se []spanEvent, e *msgp.Writer) error { kv := arrayValue{} for _, s := range se { slKeyValues := keyValueList{ - {key: 1, value: anyValue{valueType: IntValueType, value: s.TimeUnixNano}}, // time - {key: 2, value: anyValue{valueType: StringValueType, value: s.Name}}, // name + {key: streamingKey{isString: false, idx: 1}, value: anyValue{valueType: IntValueType, value: s.TimeUnixNano}}, // time + {key: streamingKey{isString: false, idx: 2}, value: anyValue{valueType: StringValueType, value: s.Name}}, // name } attr := keyValueList{} // attributes for k, v := range s.Attributes { - idx, err := encodeString(k) + idx, err := p.encodeString(k) if err != nil { - idx = k + idx = streamingKey{isString: true, stringValue: k} } attr = append(attr, keyValue{key: idx, value: anyValue{valueType: getAnyValueType(v), value: v}}) } - slKeyValues = append(slKeyValues, keyValue{key: 3, value: anyValue{valueType: ArrayValueType, value: attr}}) // attributes + slKeyValues = append(slKeyValues, keyValue{key: streamingKey{isString: false, idx: 3}, value: anyValue{valueType: ArrayValueType, value: attr}}) // attributes kv = append(kv, anyValue{valueType: keyValueListType, value: slKeyValues}) } for _, v := range kv { - err := v.EncodeMsg(e) + err := v.EncodeMsg(e, p) if err != nil { return err } From 03abab3e868f0cdcd5adcd88311d2aa54976e6da Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Tue, 23 Sep 2025 16:16:48 -0400 Subject: [PATCH 18/75] fix some types, also i don't think we need spanlistv1 anymore --- ddtrace/tracer/payload.go | 2 +- ddtrace/tracer/payload_v1.go | 14 ++++++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/ddtrace/tracer/payload.go b/ddtrace/tracer/payload.go index 7fd8c402b5..6fcca38d95 100644 --- a/ddtrace/tracer/payload.go +++ b/ddtrace/tracer/payload.go @@ -166,7 +166,7 @@ type traceChunk struct { attributes map[uint32]anyValue `msg:"attributes,omitempty"` // a list of spans in this chunk - spans spanListV1 `msg:"spans,omitempty"` + spans spanList `msg:"spans,omitempty"` // whether the trace only contains analyzed spans // (not required by tracers and set by the agent) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 894afb4fd0..d71c582b2e 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -94,7 +94,7 @@ type stringTable struct { // - uint32 // - bool // - float64 -// - uint64 +// - int64 // - uint8 // intValue(5) - 0x405 (4 indicates this is an int AnyType, then 5 is encoded using positive fixed int format) // stringValue(“a”) - 0x1a161 (1 indicates this is a string, then “a” is encoded using fixstr 0xa161) @@ -108,7 +108,7 @@ const ( StringValueType = iota + 1 // string or uint -- 1 BoolValueType // boolean -- 2 FloatValueType // float64 -- 3 - IntValueType // uint64 -- 4 + IntValueType // int64 -- 4 BytesValueType // []uint8 -- 5 ArrayValueType // []AnyValue -- 6 keyValueListType // []keyValue -- 7 @@ -133,8 +133,6 @@ type keyValue struct { type keyValueList []keyValue -type spanListV1 spanList - // newPayloadV1 returns a ready to use payloadV1. func newPayloadV1() *payloadV1 { return &payloadV1{ @@ -150,7 +148,7 @@ func newPayloadV1() *payloadV1 { } // push pushes a new item into the stream. -func (p *payloadV1) push(t spanListV1) (stats payloadStats, err error) { +func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { // We need to hydrate the payload with everything we get from the spans. // Conceptually, our `t []*Span` corresponds to one `traceChunk`. origin, priority := "", 0 @@ -173,7 +171,7 @@ func (p *payloadV1) push(t spanListV1) (stats payloadStats, err error) { traceID: t[0].Context().traceID, }) wr := msgp.NewWriter(&p.buf) - err = t.EncodeMsg(wr, p) + err = EncodeSpanList(t, wr, p) if err == nil { err = wr.Flush() } @@ -277,7 +275,7 @@ func (a *anyValue) EncodeMsg(e *msgp.Writer, p *payloadV1) error { return e.WriteFloat64(a.value.(float64)) case IntValueType: e.WriteInt32(IntValueType) - return e.WriteUint64(a.value.(uint64)) + return e.WriteInt64(a.value.(int64)) case BytesValueType: e.WriteInt32(BytesValueType) return e.WriteBytes(a.value.([]byte)) @@ -363,7 +361,7 @@ func (t *traceChunk) EncodeMsg(e *msgp.Writer, p *payloadV1) error { // Span, SpanLink, and SpanEvent structs are different for v0.4 and v1.0. // For v1 we need to manually encode the spans, span links, and span events // if we don't want to do extra allocations. -func (s spanListV1) EncodeMsg(e *msgp.Writer, p *payloadV1) error { +func EncodeSpanList(s spanList, e *msgp.Writer, p *payloadV1) error { err := e.WriteArrayHeader(uint32(len(s))) if err != nil { return msgp.WrapError(err) From 22e70ec00cd2d67878ed8c5c36d64a4dcacee1e0 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Wed, 24 Sep 2025 13:04:26 -0400 Subject: [PATCH 19/75] fix immediate compiler issues --- ddtrace/tracer/payload_v1.go | 85 ++++++++++++++++++++++++------------ 1 file changed, 56 insertions(+), 29 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index d71c582b2e..f59c516c66 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -144,6 +144,8 @@ func newPayloadV1() *payloadV1 { indices: map[string]uint32{"": 0}, nextIndex: 1, }, + header: make([]byte, 8), + off: 8, } } @@ -251,7 +253,16 @@ func (p *payloadV1) Write(data []byte) (n int, err error) { // Read implements io.Reader. It reads from the msgpack-encoded stream. func (p *payloadV1) Read(b []byte) (n int, err error) { - panic("not implemented") + if p.off < len(p.header) { + // reading header + n = copy(b, p.header[p.off:]) + p.off += n + return n, nil + } + if p.reader == nil { + p.reader = bytes.NewReader(p.buf.Bytes()) + } + return p.reader.Read(b) } // Encode the anyValue @@ -325,11 +336,7 @@ func (kv keyValueList) EncodeMsg(e *msgp.Writer, p *payloadV1) error { if err != nil { return err } - for i, k := range kv { - err := e.WriteUint32(uint32(i)) - if err != nil { - return err - } + for _, k := range kv { err = k.EncodeMsg(e, p) if err != nil { return err @@ -340,12 +347,12 @@ func (kv keyValueList) EncodeMsg(e *msgp.Writer, p *payloadV1) error { func (t *traceChunk) EncodeMsg(e *msgp.Writer, p *payloadV1) error { kv := keyValueList{ - {key: streamingKey{isString: false, idx: 1}, value: anyValue{valueType: IntValueType, value: t.priority}}, // priority - {key: streamingKey{isString: false, idx: 2}, value: anyValue{valueType: StringValueType, value: t.origin}}, // origin - {key: streamingKey{isString: false, idx: 4}, value: anyValue{valueType: keyValueListType, value: t.spans}}, // spans - {key: streamingKey{isString: false, idx: 5}, value: anyValue{valueType: BoolValueType, value: t.droppedTrace}}, // droppedTrace - {key: streamingKey{isString: false, idx: 6}, value: anyValue{valueType: BytesValueType, value: t.traceID}}, // traceID - {key: streamingKey{isString: false, idx: 7}, value: anyValue{valueType: IntValueType, value: t.samplingMechanism}}, // samplingMechanism + {key: streamingKey{isString: false, idx: 1}, value: anyValue{valueType: IntValueType, value: int64(t.priority)}}, // priority + {key: streamingKey{isString: false, idx: 2}, value: anyValue{valueType: StringValueType, value: t.origin}}, // origin + {key: streamingKey{isString: false, idx: 4}, value: anyValue{valueType: keyValueListType, value: t.spans}}, // spans + {key: streamingKey{isString: false, idx: 5}, value: anyValue{valueType: BoolValueType, value: t.droppedTrace}}, // droppedTrace + {key: streamingKey{isString: false, idx: 6}, value: anyValue{valueType: BytesValueType, value: t.traceID}}, // traceID + {key: streamingKey{isString: false, idx: 7}, value: anyValue{valueType: StringValueType, value: t.samplingMechanism}}, // samplingMechanism } attr := keyValueList{} @@ -392,14 +399,12 @@ func encodeSpan(s *Span, e *msgp.Writer, p *payloadV1) error { {key: streamingKey{isString: false, idx: 1}, value: anyValue{valueType: StringValueType, value: s.service}}, // service {key: streamingKey{isString: false, idx: 2}, value: anyValue{valueType: StringValueType, value: s.name}}, // name {key: streamingKey{isString: false, idx: 3}, value: anyValue{valueType: StringValueType, value: s.resource}}, // resource - {key: streamingKey{isString: false, idx: 4}, value: anyValue{valueType: IntValueType, value: s.spanID}}, // spanID - {key: streamingKey{isString: false, idx: 5}, value: anyValue{valueType: IntValueType, value: s.parentID}}, // parentID - {key: streamingKey{isString: false, idx: 6}, value: anyValue{valueType: IntValueType, value: s.start}}, // start - {key: streamingKey{isString: false, idx: 7}, value: anyValue{valueType: IntValueType, value: s.duration}}, // duration - {key: streamingKey{isString: false, idx: 8}, value: anyValue{valueType: BoolValueType, value: s.error}}, // error + {key: streamingKey{isString: false, idx: 4}, value: anyValue{valueType: IntValueType, value: int64(s.spanID)}}, // spanID + {key: streamingKey{isString: false, idx: 5}, value: anyValue{valueType: IntValueType, value: int64(s.parentID)}}, // parentID + {key: streamingKey{isString: false, idx: 6}, value: anyValue{valueType: IntValueType, value: int64(s.start)}}, // start + {key: streamingKey{isString: false, idx: 7}, value: anyValue{valueType: IntValueType, value: int64(s.duration)}}, // duration + {key: streamingKey{isString: false, idx: 8}, value: anyValue{valueType: BoolValueType, value: (s.error != 0)}}, // error - true if span has error {key: streamingKey{isString: false, idx: 10}, value: anyValue{valueType: StringValueType, value: s.spanType}}, // type - {key: streamingKey{isString: false, idx: 11}, value: anyValue{valueType: keyValueListType, value: s.spanLinks}}, // SpanLink - {key: streamingKey{isString: false, idx: 12}, value: anyValue{valueType: keyValueListType, value: s.spanEvents}}, // SpanEvent {key: streamingKey{isString: false, idx: 15}, value: anyValue{valueType: StringValueType, value: s.integration}}, // component } @@ -431,7 +436,7 @@ func encodeSpan(s *Span, e *msgp.Writer, p *payloadV1) error { attr = append(attr, keyValue{key: idx, value: anyValue{valueType: getAnyValueType(v), value: v}}) } - kv = append(kv, keyValue{key: streamingKey{isString: false, idx: 9}, value: anyValue{valueType: ArrayValueType, value: attr}}) // attributes + kv = append(kv, keyValue{key: streamingKey{isString: false, idx: 9}, value: anyValue{valueType: keyValueListType, value: attr}}) // attributes env, ok := s.meta["env"] if ok { @@ -442,7 +447,19 @@ func encodeSpan(s *Span, e *msgp.Writer, p *payloadV1) error { kv = append(kv, keyValue{key: streamingKey{isString: false, idx: 14}, value: anyValue{valueType: StringValueType, value: version}}) // version } - return kv.EncodeMsg(e, p) + err := kv.EncodeMsg(e, p) + if err != nil { + return err + } + + // spanLinks + err = encodeSpanLinks(s.spanLinks, e, p) + if err != nil { + return err + } + + // spanEvents + return encodeSpanEvents(s.spanEvents, e, p) } // encodeString and decodeString handles encoding a string to the payload's string table. @@ -469,8 +486,13 @@ func (p *payloadV1) encodeString(s string) (streamingKey, error) { // encodeSpanLinks encodes the span links into a msgp.Writer // Span links are represented as an array of fixmaps (keyValueList) func encodeSpanLinks(sl []SpanLink, e *msgp.Writer, p *payloadV1) error { + err := e.WriteInt32(11) // spanLinks + if err != nil { + return err + } + // write the number of span links - err := e.WriteArrayHeader(uint32(len(sl))) + err = e.WriteArrayHeader(uint32(len(sl))) if err != nil { return err } @@ -479,10 +501,10 @@ func encodeSpanLinks(sl []SpanLink, e *msgp.Writer, p *payloadV1) error { kv := arrayValue{} for _, s := range sl { slKeyValues := keyValueList{ - {key: streamingKey{isString: false, idx: 1}, value: anyValue{valueType: IntValueType, value: s.TraceID}}, // traceID - {key: streamingKey{isString: false, idx: 2}, value: anyValue{valueType: IntValueType, value: s.SpanID}}, // spanID - {key: streamingKey{isString: false, idx: 4}, value: anyValue{valueType: StringValueType, value: s.Tracestate}}, // tracestate - {key: streamingKey{isString: false, idx: 5}, value: anyValue{valueType: IntValueType, value: s.Flags}}, // flags + {key: streamingKey{isString: false, idx: 1}, value: anyValue{valueType: IntValueType, value: int64(s.TraceID)}}, // traceID + {key: streamingKey{isString: false, idx: 2}, value: anyValue{valueType: IntValueType, value: int64(s.SpanID)}}, // spanID + {key: streamingKey{isString: false, idx: 4}, value: anyValue{valueType: StringValueType, value: s.Tracestate}}, // tracestate + {key: streamingKey{isString: false, idx: 5}, value: anyValue{valueType: IntValueType, value: int64(s.Flags)}}, // flags } attr := keyValueList{} @@ -510,8 +532,13 @@ func encodeSpanLinks(sl []SpanLink, e *msgp.Writer, p *payloadV1) error { // encodeSpanEvents encodes the span events into a msgp.Writer // Span events are represented as an array of fixmaps (keyValueList) func encodeSpanEvents(se []spanEvent, e *msgp.Writer, p *payloadV1) error { + err := e.WriteInt32(12) // spanEvents + if err != nil { + return err + } + // write the number of span events - err := e.WriteArrayHeader(uint32(len(se))) + err = e.WriteArrayHeader(uint32(len(se))) if err != nil { return err } @@ -520,8 +547,8 @@ func encodeSpanEvents(se []spanEvent, e *msgp.Writer, p *payloadV1) error { kv := arrayValue{} for _, s := range se { slKeyValues := keyValueList{ - {key: streamingKey{isString: false, idx: 1}, value: anyValue{valueType: IntValueType, value: s.TimeUnixNano}}, // time - {key: streamingKey{isString: false, idx: 2}, value: anyValue{valueType: StringValueType, value: s.Name}}, // name + {key: streamingKey{isString: false, idx: 1}, value: anyValue{valueType: IntValueType, value: int64(s.TimeUnixNano)}}, // time + {key: streamingKey{isString: false, idx: 2}, value: anyValue{valueType: StringValueType, value: s.Name}}, // name } attr := keyValueList{} From 68ab0d84a8432e26b32b507ec72540a22319d24d Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Wed, 24 Sep 2025 14:40:53 -0400 Subject: [PATCH 20/75] few more fixes with payload representations --- ddtrace/tracer/payload.go | 2 +- ddtrace/tracer/payload_v1.go | 48 ++++++++++++++++++++++++------------ 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/ddtrace/tracer/payload.go b/ddtrace/tracer/payload.go index 6fcca38d95..2532a53943 100644 --- a/ddtrace/tracer/payload.go +++ b/ddtrace/tracer/payload.go @@ -163,7 +163,7 @@ type traceChunk struct { origin string `msg:"origin,omitempty"` // a collection of key to value pairs common in all `spans` - attributes map[uint32]anyValue `msg:"attributes,omitempty"` + attributes keyValueList `msg:"attributes,omitempty"` // a list of spans in this chunk spans spanList `msg:"spans,omitempty"` diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index f59c516c66..610d543864 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -34,31 +34,31 @@ type payloadV1 struct { strings stringTable `msgp:"strings"` // the string ID of the container where the tracer is running - containerID uint32 `msgp:"containerID"` + containerID string `msgp:"containerID"` // the string language name of the tracer - languageName uint32 `msgp:"languageName"` + languageName string `msgp:"languageName"` // the string language version of the tracer - languageVersion uint32 `msgp:"languageVersion"` + languageVersion string `msgp:"languageVersion"` // the string version of the tracer - tracerVersion uint32 `msgp:"tracerVersion"` + tracerVersion string `msgp:"tracerVersion"` // the V4 string UUID representation of a tracer session - runtimeID uint32 `msgp:"runtimeID"` + runtimeID string `msgp:"runtimeID"` // the optional `env` string tag that set with the tracer - env uint32 `msgp:"env,omitempty"` + env string `msgp:"env,omitempty"` // the optional string hostname of where the tracer is running - hostname uint32 `msgp:"hostname,omitempty"` + hostname string `msgp:"hostname,omitempty"` // the optional string `version` tag for the application set in the tracer - appVersion uint32 `msgp:"appVersion,omitempty"` + appVersion string `msgp:"appVersion,omitempty"` // a collection of key to value pairs common in all `chunks` - attributes map[uint32]anyValue `msgp:"attributes,omitempty"` + attributes keyValueList `msgp:"attributes,omitempty"` // a list of trace `chunks` chunks []traceChunk `msgp:"chunks,omitempty"` @@ -137,7 +137,7 @@ type keyValueList []keyValue func newPayloadV1() *payloadV1 { return &payloadV1{ protocolVersion: traceProtocolV1, - attributes: make(map[uint32]anyValue), + attributes: keyValueList{}, chunks: make([]traceChunk, 0), strings: stringTable{ strings: []string{""}, @@ -165,23 +165,39 @@ func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { } } + kv := keyValueList{ + {key: streamingKey{isString: false, idx: 2}, value: anyValue{valueType: StringValueType, value: p.containerID}}, // containerID + {key: streamingKey{isString: false, idx: 3}, value: anyValue{valueType: StringValueType, value: p.languageName}}, // languageName + {key: streamingKey{isString: false, idx: 4}, value: anyValue{valueType: StringValueType, value: p.languageVersion}}, // languageVersion + {key: streamingKey{isString: false, idx: 5}, value: anyValue{valueType: StringValueType, value: p.tracerVersion}}, // tracerVersion + {key: streamingKey{isString: false, idx: 6}, value: anyValue{valueType: StringValueType, value: p.runtimeID}}, // runtimeID + {key: streamingKey{isString: false, idx: 7}, value: anyValue{valueType: StringValueType, value: p.env}}, // env + {key: streamingKey{isString: false, idx: 8}, value: anyValue{valueType: StringValueType, value: p.hostname}}, // hostname + {key: streamingKey{isString: false, idx: 9}, value: anyValue{valueType: StringValueType, value: p.appVersion}}, // appVersion + } + p.chunks = append(p.chunks, traceChunk{ priority: int32(priority), origin: origin, - attributes: make(map[uint32]anyValue), + attributes: keyValueList{}, spans: t, traceID: t[0].Context().traceID, }) wr := msgp.NewWriter(&p.buf) err = EncodeSpanList(t, wr, p) - if err == nil { - err = wr.Flush() - } if err != nil { return payloadStats{}, err } + + // once we've encoded the spans, we can encode the attributes + kv = append(kv, keyValue{key: streamingKey{isString: false, idx: 10}, value: anyValue{valueType: keyValueListType, value: p.attributes}}) // attributes + err = kv.EncodeMsg(wr, p) + if err == nil { + err = wr.Flush() + } + p.recordItem() - return p.stats(), nil + return p.stats(), err } func (p *payloadV1) grow(n int) { @@ -357,7 +373,7 @@ func (t *traceChunk) EncodeMsg(e *msgp.Writer, p *payloadV1) error { attr := keyValueList{} for k, v := range t.attributes { - attr = append(attr, keyValue{key: streamingKey{isString: false, idx: k}, value: anyValue{valueType: getAnyValueType(v), value: v}}) + attr = append(attr, keyValue{key: streamingKey{isString: false, idx: uint32(k)}, value: anyValue{valueType: getAnyValueType(v), value: v}}) } kv = append(kv, keyValue{key: streamingKey{isString: false, idx: 3}, value: anyValue{valueType: ArrayValueType, value: attr}}) // attributes From 23c72cfc80c29aca3afd4e6ac1936e37fac76053 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Mon, 29 Sep 2025 13:36:26 -0400 Subject: [PATCH 21/75] wip: decoding functions --- ddtrace/tracer/payload_v1.go | 548 ++++++++++++++++++++++++++++++++++- 1 file changed, 536 insertions(+), 12 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 610d543864..8b82fa7c03 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -11,6 +11,7 @@ import ( "fmt" "sync/atomic" + "github.com/DataDog/dd-trace-go/v2/ddtrace/ext" "github.com/tinylib/msgp/msgp" ) @@ -31,7 +32,7 @@ type payloadV1 struct { // array of strings referenced in this tracer payload, its chunks and spans // stringTable holds references from a string value to an index. // the 0th position in the stringTable should always be the empty string. - strings stringTable `msgp:"strings"` + strings *stringTable `msgp:"strings"` // the string ID of the container where the tracer is running containerID string `msgp:"containerID"` @@ -139,16 +140,29 @@ func newPayloadV1() *payloadV1 { protocolVersion: traceProtocolV1, attributes: keyValueList{}, chunks: make([]traceChunk, 0), - strings: stringTable{ - strings: []string{""}, - indices: map[string]uint32{"": 0}, - nextIndex: 1, - }, - header: make([]byte, 8), - off: 8, + strings: newStringTable(), + header: make([]byte, 8), + off: 8, } } +func newStringTable() *stringTable { + return &stringTable{ + strings: []string{""}, + indices: map[string]uint32{"": 0}, + nextIndex: 1, + } +} + +func (s *stringTable) Add(str string) { + if _, ok := s.indices[str]; ok { + return + } + s.indices[str] = s.nextIndex + s.strings = append(s.strings, str) + s.nextIndex += 1 +} + // push pushes a new item into the stream. func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { // We need to hydrate the payload with everything we get from the spans. @@ -484,7 +498,7 @@ func encodeSpan(s *Span, e *msgp.Writer, p *payloadV1) error { // - otherwise, write the string into the message, then add the string at the next index // Returns the index of the string in the string table, and an error if there is one func (p *payloadV1) encodeString(s string) (streamingKey, error) { - sTable := &p.strings + sTable := p.strings idx, ok := sTable.indices[s] // if the string already exists in the table, use its index if ok { @@ -493,9 +507,7 @@ func (p *payloadV1) encodeString(s string) (streamingKey, error) { // else, write the string into the table at the next index // return an error to indicate that the string should be written to the msgp message - sTable.indices[s] = sTable.nextIndex - sTable.strings = append(sTable.strings, s) - sTable.nextIndex += 1 + sTable.Add(s) return streamingKey{isString: true, stringValue: s}, nil } @@ -604,3 +616,515 @@ func getAnyValueType(v any) int { } return IntValueType } + +func (p *payloadV1) Decode(b []byte) ([]byte, error) { + if p.strings == nil { + p.strings = newStringTable() + } + + fields, o, err := msgp.ReadMapHeaderBytes(b) + if err != nil { + return o, err + } + + for fields > 0 { + fields-- + + f, o, err := msgp.ReadUint32Bytes(b) + if err != nil { + return o, err + } + + switch f { + case 1: // stringTable + o, err = DecodeStringTable(o, p.strings) + if err != nil { + return o, err + } + + case 2: // containerID + p.containerID, o, err = DecodeStreamingString(o, p.strings) + if err != nil { + return o, err + } + + case 3: // languageName + p.languageName, o, err = DecodeStreamingString(o, p.strings) + if err != nil { + return o, err + } + + case 4: // languageVersion + p.languageVersion, o, err = DecodeStreamingString(o, p.strings) + if err != nil { + return o, err + } + + case 5: // tracerVersion + p.tracerVersion, o, err = DecodeStreamingString(o, p.strings) + if err != nil { + return o, err + } + + case 6: // runtimeID + p.runtimeID, o, err = DecodeStreamingString(o, p.strings) + if err != nil { + return o, err + } + + case 7: // env + p.env, o, err = DecodeStreamingString(o, p.strings) + if err != nil { + return o, err + } + + case 8: // hostname + p.hostname, o, err = DecodeStreamingString(o, p.strings) + if err != nil { + return o, err + } + + case 9: // appVersion + p.appVersion, o, err = DecodeStreamingString(o, p.strings) + if err != nil { + return o, err + } + case 10: // attributes + p.attributes, o, err = DecodeKeyValueList(o, p.strings) + if err != nil { + return o, err + } + case 11: // chunks + p.chunks, o, err = DecodeTraceChunks(o, p.strings) + if err != nil { + return o, err + } + } + } + return o, nil +} + +func DecodeStringTable(b []byte, strings *stringTable) ([]byte, error) { + len, o, err := msgp.ReadBytesHeader(b) + if err != nil { + return nil, err + } + + for len > 0 { + len-- + str, o, err := msgp.ReadStringBytes(o) + if err != nil { + return o, err + } + + // if we've seen the string before, skip + if _, ok := strings.indices[str]; ok { + continue + } + + strings.Add(str) + } + return o, nil +} + +func DecodeStreamingString(b []byte, strings *stringTable) (string, []byte, error) { + if len(b) == 0 { + return "", nil, msgp.WrapError(nil, "expected streaming string, got EOF") + } + // try reading as a uint32 index + idx, o, err := msgp.ReadUint32Bytes(b) + if err == nil { + return strings.strings[idx], o, nil + } + + // else, try reading as a string, then add to the string table + str, o, err := msgp.ReadStringBytes(b) + if err != nil { + return "", nil, msgp.WrapError(err, "unable to read streaming string") + } + strings.Add(str) + return str, o, nil +} + +func DecodeAnyValue(b []byte, strings *stringTable) (anyValue, []byte, error) { + vType, o, err := msgp.ReadInt32Bytes(b) + if err != nil { + return anyValue{}, o, err + } + switch vType { + case StringValueType: + str, o, err := msgp.ReadStringBytes(o) + if err != nil { + return anyValue{}, o, err + } + return anyValue{valueType: StringValueType, value: str}, o, nil + case BoolValueType: + b, o, err := msgp.ReadBoolBytes(o) + if err != nil { + return anyValue{}, o, err + } + return anyValue{valueType: BoolValueType, value: b}, o, nil + case FloatValueType: + f, o, err := msgp.ReadFloat64Bytes(o) + if err != nil { + return anyValue{}, o, err + } + return anyValue{valueType: FloatValueType, value: f}, o, nil + case IntValueType: + i, o, err := msgp.ReadInt64Bytes(o) + if err != nil { + return anyValue{}, o, err + } + return anyValue{valueType: IntValueType, value: i}, o, nil + case BytesValueType: + b, o, err := msgp.ReadBytesBytes(o, nil) + if err != nil { + return anyValue{}, o, err + } + return anyValue{valueType: BytesValueType, value: b}, o, nil + case ArrayValueType: + len, o, err := msgp.ReadBytesHeader(o) + if err != nil { + return anyValue{}, o, err + } + arrayValue := make(arrayValue, len/2) + for i := range len / 2 { + arrayValue[i], o, err = DecodeAnyValue(o, strings) + if err != nil { + return anyValue{}, o, err + } + } + return anyValue{valueType: ArrayValueType, value: arrayValue}, o, nil + case keyValueListType: + kv, o, err := DecodeKeyValueList(o, strings) + if err != nil { + return anyValue{}, o, err + } + return anyValue{valueType: keyValueListType, value: kv}, o, nil + default: + return anyValue{}, o, fmt.Errorf("invalid value type: %d", vType) + } +} + +func DecodeKeyValueList(b []byte, strings *stringTable) (keyValueList, []byte, error) { + len, o, err := msgp.ReadBytesHeader(b) + if err != nil { + return nil, o, err + } + + if len == 0 || len%3 != 0 { + return nil, o, msgp.WrapError(fmt.Errorf("invalid number of items in keyValueList encoding, expected multiple of 3, got %d", len)) + } + + kv := make(keyValueList, len/3) + for i := range len / 3 { + len-- + key, o, err := DecodeStreamingString(o, strings) + if err != nil { + return nil, o, err + } + v, o, err := DecodeAnyValue(o, strings) + if err != nil { + return nil, o, err + } + kv[i] = keyValue{key: streamingKey{isString: true, stringValue: key}, value: anyValue{valueType: v.valueType, value: v.value}} + } + return kv, o, nil +} + +func DecodeTraceChunks(b []byte, strings *stringTable) ([]traceChunk, []byte, error) { + tc := []traceChunk{} + fields, o, err := msgp.ReadMapHeaderBytes(b) + if err != nil { + return tc, o, err + } + + for fields > 0 { + fields-- + + f, o, err := msgp.ReadUint32Bytes(b) + if err != nil { + return tc, o, err + } + + switch f { + case 1: // priority + + case 2: // origin + + case 3: // attributes + + case 4: // spans + + case 5: // droppedTrace + + case 6: // traceID + + case 7: // samplingMechanism + } + } + return tc, o, nil +} + +func DecodeSpan(b []byte, strings *stringTable) (*Span, []byte, error) { + sp := Span{} + fields, o, err := msgp.ReadMapHeaderBytes(b) + if err != nil { + return &sp, o, err + } + + for fields > 0 { + fields-- + + f, o, err := msgp.ReadUint32Bytes(b) + if err != nil { + return &sp, o, err + } + + switch f { + case 1: // service + st, o, err := msgp.ReadStringBytes(o) + if err != nil { + return &sp, o, err + } + sp.service = st + case 2: // name + st, o, err := msgp.ReadStringBytes(o) + if err != nil { + return &sp, o, err + } + sp.name = st + case 3: // resource + st, o, err := msgp.ReadStringBytes(o) + if err != nil { + return &sp, o, err + } + sp.resource = st + case 4: // spanID + i, o, err := msgp.ReadInt64Bytes(o) + if err != nil { + return &sp, o, err + } + sp.spanID = uint64(i) + case 5: // parentID + i, o, err := msgp.ReadInt64Bytes(o) + if err != nil { + return &sp, o, err + } + sp.parentID = uint64(i) + case 6: // start + i, o, err := msgp.ReadInt64Bytes(o) + if err != nil { + return &sp, o, err + } + sp.start = i + case 7: // duration + i, o, err := msgp.ReadInt64Bytes(o) + if err != nil { + return &sp, o, err + } + sp.duration = i + case 8: // error + i, o, err := msgp.ReadBoolBytes(o) + if err != nil { + return &sp, o, err + } + if i { + sp.error = 1 + } else { + sp.error = 0 + } + case 9: // attributes + kv, o, err := DecodeKeyValueList(o, strings) + if err != nil { + return &sp, o, err + } + for k, v := range kv { + key := strings.strings[k] + sp.SetTag(key, v.value.value) + } + case 10: // type + st, o, err := msgp.ReadStringBytes(o) + if err != nil { + return &sp, o, err + } + sp.spanType = st + case 11: // spanLinks + sl, o, err := DecodeSpanLinks(o, strings) + if err != nil { + return &sp, o, err + } + sp.spanLinks = sl + case 12: // spanEvents + se, o, err := DecodeSpanEvents(o, strings) + if err != nil { + return &sp, o, err + } + sp.spanEvents = se + case 13: // env + s, o, err := msgp.ReadStringBytes(o) + if err != nil { + return &sp, o, err + } + sp.SetTag(ext.Environment, s) + case 14: // version + s, o, err := msgp.ReadStringBytes(o) + if err != nil { + return &sp, o, err + } + sp.setMeta(ext.Version, s) + case 15: // component + s, o, err := msgp.ReadStringBytes(o) + if err != nil { + return &sp, o, err + } + sp.integration = s + } + } + return &sp, nil, nil +} + +func DecodeSpanLinks(b []byte, strings *stringTable) ([]SpanLink, []byte, error) { + numSpanLinks, o, err := msgp.ReadArrayHeaderBytes(b) + if err != nil { + return nil, o, err + } + + ret := make([]SpanLink, numSpanLinks) + for i := range numSpanLinks { + sl := SpanLink{} + fields, o, err := msgp.ReadMapHeaderBytes(o) + if err != nil { + return ret, o, err + } + for fields > 0 { + fields-- + + f, o, err := msgp.ReadUint32Bytes(o) + if err != nil { + return ret, o, err + } + + switch f { + case 1: // traceID + s, o, err := msgp.ReadInt64Bytes(o) + if err != nil { + return ret, o, err + } + sl.TraceID = uint64(s) + case 2: // spanID + s, o, err := msgp.ReadInt64Bytes(o) + if err != nil { + return ret, o, err + } + sl.SpanID = uint64(s) + case 3: // attributes + kv, o, err := DecodeKeyValueList(o, strings) + if err != nil { + return ret, o, err + } + for k, v := range kv { + key := strings.strings[k] + s, ok := v.value.value.(string) + if !ok { + err := msgp.WrapError(fmt.Errorf("expected string value type for span link attributes, got %T", v.value.value)) + return ret, o, err + } + sl.Attributes[key] = s + } + case 4: // tracestate + s, o, err := msgp.ReadStringBytes(o) + if err != nil { + return ret, o, err + } + sl.Tracestate = s + case 5: // flags + s, o, err := msgp.ReadInt32Bytes(o) + if err != nil { + return ret, o, err + } + sl.Flags = uint32(s) + } + } + ret[i] = sl + } + return ret, o, nil +} + +func DecodeSpanEvents(b []byte, strings *stringTable) ([]spanEvent, []byte, error) { + numSpanEvents, o, err := msgp.ReadArrayHeaderBytes(b) + if err != nil { + return nil, o, err + } + ret := make([]spanEvent, numSpanEvents) + for i := range numSpanEvents { + se := spanEvent{} + fields, o, err := msgp.ReadMapHeaderBytes(o) + if err != nil { + return ret, o, err + } + for fields > 0 { + fields-- + + f, o, err := msgp.ReadUint32Bytes(b) + if err != nil { + return ret, o, err + } + + switch f { + case 1: // time + s, o, err := msgp.ReadInt64Bytes(o) + if err != nil { + return ret, o, err + } + se.TimeUnixNano = uint64(s) + case 2: // name + s, o, err := msgp.ReadStringBytes(o) + if err != nil { + return ret, o, err + } + se.Name = s + case 4: // attributes + kv, o, err := DecodeKeyValueList(o, strings) + if err != nil { + return ret, o, err + } + for k, v := range kv { + key := strings.strings[k] + switch v.value.valueType { + case StringValueType: + se.Attributes[key] = &spanEventAttribute{ + Type: spanEventAttributeTypeString, + StringValue: v.value.value.(string), + } + case BoolValueType: + se.Attributes[key] = &spanEventAttribute{ + Type: spanEventAttributeTypeBool, + BoolValue: v.value.value.(bool), + } + case IntValueType: + se.Attributes[key] = &spanEventAttribute{ + Type: spanEventAttributeTypeInt, + IntValue: v.value.value.(int64), + } + case FloatValueType: + se.Attributes[key] = &spanEventAttribute{ + Type: spanEventAttributeTypeDouble, + DoubleValue: v.value.value.(float64), + } + case ArrayValueType: + se.Attributes[key] = &spanEventAttribute{ + Type: spanEventAttributeTypeArray, + ArrayValue: v.value.value.(*spanEventArrayAttribute), + } + default: + err := msgp.WrapError(fmt.Errorf("unexpected value type not supported by span events: %T", v.value.value)) + return ret, o, err + } + } + } + } + ret[i] = se + } + return ret, nil, nil +} From a12502c88158fd1c6abc6ee2dd88d1c6ee2d5fcf Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Mon, 29 Sep 2025 14:54:05 -0400 Subject: [PATCH 22/75] wip: decode trace chunk func --- ddtrace/tracer/payload_v1.go | 94 ++++++++++++++++++++++++++++-------- 1 file changed, 73 insertions(+), 21 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 8b82fa7c03..84f13fa83f 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -833,37 +833,89 @@ func DecodeKeyValueList(b []byte, strings *stringTable) (keyValueList, []byte, e } func DecodeTraceChunks(b []byte, strings *stringTable) ([]traceChunk, []byte, error) { - tc := []traceChunk{} - fields, o, err := msgp.ReadMapHeaderBytes(b) + len, o, err := msgp.ReadArrayHeaderBytes(b) if err != nil { - return tc, o, err + return nil, o, err } - for fields > 0 { - fields-- - - f, o, err := msgp.ReadUint32Bytes(b) + ret := make([]traceChunk, len) + for i := range len { + fields, o, err := msgp.ReadMapHeaderBytes(o) if err != nil { - return tc, o, err + return nil, o, err } + tc := traceChunk{} + for fields > 0 { + fields-- - switch f { - case 1: // priority - - case 2: // origin - - case 3: // attributes - - case 4: // spans - - case 5: // droppedTrace + f, o, err := msgp.ReadUint32Bytes(b) + if err != nil { + return ret, o, err + } - case 6: // traceID + switch f { + case 1: // priority + s, o, err := msgp.ReadInt32Bytes(o) + if err != nil { + return ret, o, err + } + tc.priority = s + case 2: // origin + s, o, err := msgp.ReadStringBytes(o) + if err != nil { + return ret, o, err + } + tc.origin = s + case 3: // attributes + kv, o, err := DecodeKeyValueList(o, strings) + if err != nil { + return ret, o, err + } + tc.attributes = kv + case 4: // spans + s, o, err := DecodeSpanList(o, strings) + if err != nil { + return ret, o, err + } + tc.spans = s + case 5: // droppedTrace + s, o, err := msgp.ReadBoolBytes(o) + if err != nil { + return ret, o, err + } + tc.droppedTrace = s + case 6: // traceID + s, o, err := msgp.ReadBytesBytes(o, nil) + if err != nil { + return ret, o, err + } + tc.traceID = [16]byte(s) + case 7: // samplingMechanism + s, o, err := msgp.ReadStringBytes(o) + if err != nil { + return ret, o, err + } + tc.samplingMechanism = s + } + } + ret[i] = tc + } + return ret, o, nil +} - case 7: // samplingMechanism +func DecodeSpanList(b []byte, strings *stringTable) (spanList, []byte, error) { + len, o, err := msgp.ReadArrayHeaderBytes(b) + if err != nil { + return nil, o, err + } + ret := make([]*Span, len) + for i := range len { + ret[i], o, err = DecodeSpan(o, strings) + if err != nil { + return nil, o, err } } - return tc, o, nil + return ret, o, nil } func DecodeSpan(b []byte, strings *stringTable) (*Span, []byte, error) { From 08b1b9773704cedd81a59a74589576f95b2e0e30 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Tue, 30 Sep 2025 13:13:52 -0400 Subject: [PATCH 23/75] couple fixes --- ddtrace/tracer/payload.go | 2 +- ddtrace/tracer/payload_v1.go | 62 ++++++++++++++++++------------------ 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/ddtrace/tracer/payload.go b/ddtrace/tracer/payload.go index 2532a53943..70be2dd865 100644 --- a/ddtrace/tracer/payload.go +++ b/ddtrace/tracer/payload.go @@ -173,7 +173,7 @@ type traceChunk struct { droppedTrace bool `msg:"droppedTrace"` // the ID of the trace to which all spans in this chunk belong - traceID [16]byte `msg:"traceID"` + traceID []byte `msg:"traceID"` // the optional string decision maker (previously span tag _dd.p.dm) samplingMechanism string `msg:"samplingMechanism,omitempty"` diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 84f13fa83f..a8044a06e8 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -190,15 +190,17 @@ func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { {key: streamingKey{isString: false, idx: 9}, value: anyValue{valueType: StringValueType, value: p.appVersion}}, // appVersion } - p.chunks = append(p.chunks, traceChunk{ + tc := traceChunk{ priority: int32(priority), origin: origin, attributes: keyValueList{}, spans: t, - traceID: t[0].Context().traceID, - }) + traceID: t[0].Context().traceID[:], + } + p.chunks = append(p.chunks, tc) wr := msgp.NewWriter(&p.buf) - err = EncodeSpanList(t, wr, p) + + err = tc.EncodeMsg(wr, p) if err != nil { return payloadStats{}, err } @@ -376,10 +378,11 @@ func (kv keyValueList) EncodeMsg(e *msgp.Writer, p *payloadV1) error { } func (t *traceChunk) EncodeMsg(e *msgp.Writer, p *payloadV1) error { + e.WriteInt32(11) // write msgp index for `chunks` + kv := keyValueList{ {key: streamingKey{isString: false, idx: 1}, value: anyValue{valueType: IntValueType, value: int64(t.priority)}}, // priority {key: streamingKey{isString: false, idx: 2}, value: anyValue{valueType: StringValueType, value: t.origin}}, // origin - {key: streamingKey{isString: false, idx: 4}, value: anyValue{valueType: keyValueListType, value: t.spans}}, // spans {key: streamingKey{isString: false, idx: 5}, value: anyValue{valueType: BoolValueType, value: t.droppedTrace}}, // droppedTrace {key: streamingKey{isString: false, idx: 6}, value: anyValue{valueType: BytesValueType, value: t.traceID}}, // traceID {key: streamingKey{isString: false, idx: 7}, value: anyValue{valueType: StringValueType, value: t.samplingMechanism}}, // samplingMechanism @@ -389,22 +392,24 @@ func (t *traceChunk) EncodeMsg(e *msgp.Writer, p *payloadV1) error { for k, v := range t.attributes { attr = append(attr, keyValue{key: streamingKey{isString: false, idx: uint32(k)}, value: anyValue{valueType: getAnyValueType(v), value: v}}) } - kv = append(kv, keyValue{key: streamingKey{isString: false, idx: 3}, value: anyValue{valueType: ArrayValueType, value: attr}}) // attributes + kv = append(kv, keyValue{key: streamingKey{isString: false, idx: 3}, value: anyValue{valueType: keyValueListType, value: attr}}) // attributes + + err := kv.EncodeMsg(e, p) + if err != nil { + return err + } - return kv.EncodeMsg(e, p) + return EncodeSpanList(t.spans, e, p) } -// EncodeMsg writes the contents of a list of spans into `p.buf` -// Span, SpanLink, and SpanEvent structs are different for v0.4 and v1.0. -// For v1 we need to manually encode the spans, span links, and span events -// if we don't want to do extra allocations. func EncodeSpanList(s spanList, e *msgp.Writer, p *payloadV1) error { + e.WriteInt32(4) // write msgp index for `spans` + err := e.WriteArrayHeader(uint32(len(s))) if err != nil { return msgp.WrapError(err) } - e.WriteInt32(4) for _, span := range s { if span == nil { err := e.WriteNil() @@ -622,7 +627,7 @@ func (p *payloadV1) Decode(b []byte) ([]byte, error) { p.strings = newStringTable() } - fields, o, err := msgp.ReadMapHeaderBytes(b) + fields, o, err := msgp.ReadArrayHeaderBytes(b) if err != nil { return o, err } @@ -630,18 +635,13 @@ func (p *payloadV1) Decode(b []byte) ([]byte, error) { for fields > 0 { fields-- - f, o, err := msgp.ReadUint32Bytes(b) + f, o, err := msgp.ReadInt32Bytes(o) if err != nil { return o, err } switch f { - case 1: // stringTable - o, err = DecodeStringTable(o, p.strings) - if err != nil { - return o, err - } - + // we don't care for the string table, so we don't decode it case 2: // containerID p.containerID, o, err = DecodeStreamingString(o, p.strings) if err != nil { @@ -738,7 +738,7 @@ func DecodeStreamingString(b []byte, strings *stringTable) (string, []byte, erro } // else, try reading as a string, then add to the string table - str, o, err := msgp.ReadStringBytes(b) + str, o, err := msgp.ReadStringBytes(o) if err != nil { return "", nil, msgp.WrapError(err, "unable to read streaming string") } @@ -783,7 +783,7 @@ func DecodeAnyValue(b []byte, strings *stringTable) (anyValue, []byte, error) { } return anyValue{valueType: BytesValueType, value: b}, o, nil case ArrayValueType: - len, o, err := msgp.ReadBytesHeader(o) + len, o, err := msgp.ReadArrayHeaderBytes(o) if err != nil { return anyValue{}, o, err } @@ -807,7 +807,7 @@ func DecodeAnyValue(b []byte, strings *stringTable) (anyValue, []byte, error) { } func DecodeKeyValueList(b []byte, strings *stringTable) (keyValueList, []byte, error) { - len, o, err := msgp.ReadBytesHeader(b) + len, o, err := msgp.ReadMapHeaderBytes(b) if err != nil { return nil, o, err } @@ -833,14 +833,14 @@ func DecodeKeyValueList(b []byte, strings *stringTable) (keyValueList, []byte, e } func DecodeTraceChunks(b []byte, strings *stringTable) ([]traceChunk, []byte, error) { - len, o, err := msgp.ReadArrayHeaderBytes(b) + len, o, err := msgp.ReadMapHeaderBytes(b) if err != nil { return nil, o, err } ret := make([]traceChunk, len) for i := range len { - fields, o, err := msgp.ReadMapHeaderBytes(o) + fields, o, err := msgp.ReadArrayHeaderBytes(o) if err != nil { return nil, o, err } @@ -848,7 +848,7 @@ func DecodeTraceChunks(b []byte, strings *stringTable) ([]traceChunk, []byte, er for fields > 0 { fields-- - f, o, err := msgp.ReadUint32Bytes(b) + f, o, err := msgp.ReadUint32Bytes(o) if err != nil { return ret, o, err } @@ -889,7 +889,7 @@ func DecodeTraceChunks(b []byte, strings *stringTable) ([]traceChunk, []byte, er if err != nil { return ret, o, err } - tc.traceID = [16]byte(s) + tc.traceID = []byte(s) case 7: // samplingMechanism s, o, err := msgp.ReadStringBytes(o) if err != nil { @@ -928,7 +928,7 @@ func DecodeSpan(b []byte, strings *stringTable) (*Span, []byte, error) { for fields > 0 { fields-- - f, o, err := msgp.ReadUint32Bytes(b) + f, o, err := msgp.ReadUint32Bytes(o) if err != nil { return &sp, o, err } @@ -1091,11 +1091,11 @@ func DecodeSpanLinks(b []byte, strings *stringTable) ([]SpanLink, []byte, error) } sl.Tracestate = s case 5: // flags - s, o, err := msgp.ReadInt32Bytes(o) + s, o, err := msgp.ReadUint32Bytes(o) if err != nil { return ret, o, err } - sl.Flags = uint32(s) + sl.Flags = s } } ret[i] = sl @@ -1118,7 +1118,7 @@ func DecodeSpanEvents(b []byte, strings *stringTable) ([]spanEvent, []byte, erro for fields > 0 { fields-- - f, o, err := msgp.ReadUint32Bytes(b) + f, o, err := msgp.ReadUint32Bytes(o) if err != nil { return ret, o, err } From 60165d3ba2756bf671a5497a65f2cd3058bef2ac Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Tue, 30 Sep 2025 13:35:58 -0400 Subject: [PATCH 24/75] draft test --- ddtrace/tracer/payload_test.go | 44 ++++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/ddtrace/tracer/payload_test.go b/ddtrace/tracer/payload_test.go index 239ca52385..4c840904b5 100644 --- a/ddtrace/tracer/payload_test.go +++ b/ddtrace/tracer/payload_test.go @@ -60,9 +60,9 @@ func TestPayloadIntegrity(t *testing.T) { } } -// TestPayloadDecode ensures that whatever we push into the payload can +// TestPayloadV04Decode ensures that whatever we push into a v0.4 payload can // be decoded by the codec. -func TestPayloadDecode(t *testing.T) { +func TestPayloadV04Decode(t *testing.T) { for _, n := range []int{10, 1 << 10} { t.Run(strconv.Itoa(n), func(t *testing.T) { assert := assert.New(t) @@ -77,6 +77,46 @@ func TestPayloadDecode(t *testing.T) { } } +// TestPayloadV1Decode ensures that whatever we push into a v1 payload can +// be decoded by the codec, and that it matches the original payload. +func TestPayloadV1Decode(t *testing.T) { + for _, n := range []int{10, 1 << 10} { + t.Run(strconv.Itoa(n), func(t *testing.T) { + assert := assert.New(t) + p := newPayloadV1() + + p.containerID = "containerID" + p.languageName = "languageName" + p.languageVersion = "languageVersion" + p.tracerVersion = "tracerVersion" + p.runtimeID = "runtimeID" + p.env = "env" + p.hostname = "hostname" + p.appVersion = "appVersion" + + for i := 0; i < n; i++ { + _, _ = p.push(newSpanList(i%5 + 1)) + } + + encoded, err := io.ReadAll(p) + assert.NoError(err) + + got := newPayloadV1() + _, err = got.Decode(encoded) + assert.NoError(err) + + assert.Equal(p.containerID, got.containerID) + assert.Equal(p.languageName, got.languageName) + assert.Equal(p.languageVersion, got.languageVersion) + assert.Equal(p.tracerVersion, got.tracerVersion) + assert.Equal(p.runtimeID, got.runtimeID) + assert.Equal(p.env, got.env) + assert.Equal(p.hostname, got.hostname) + assert.Equal(p.appVersion, got.appVersion) + }) + } +} + func BenchmarkPayloadThroughput(b *testing.B) { b.Run("10K", benchmarkPayloadThroughput(1)) b.Run("100K", benchmarkPayloadThroughput(10)) From 11a4ad9046b39e10b8ec3be6d7ead9087bdede27 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Tue, 7 Oct 2025 17:51:39 -0400 Subject: [PATCH 25/75] wip with many questions --- ddtrace/tracer/payload.go | 32 +- ddtrace/tracer/payload_test.go | 123 ++- ddtrace/tracer/payload_v1.go | 1422 +++++++++++++------------------- 3 files changed, 681 insertions(+), 896 deletions(-) diff --git a/ddtrace/tracer/payload.go b/ddtrace/tracer/payload.go index 70be2dd865..8d8d04979e 100644 --- a/ddtrace/tracer/payload.go +++ b/ddtrace/tracer/payload.go @@ -60,9 +60,15 @@ func newPayload(protocol float64) payload { // https://github.com/msgpack/msgpack/blob/master/spec.md#array-format-family const ( + // arrays msgpackArrayFix byte = 144 // up to 15 items msgpackArray16 byte = 0xdc // up to 2^16-1 items, followed by size in 2 bytes msgpackArray32 byte = 0xdd // up to 2^32-1 items, followed by size in 4 bytes + + // maps + msgpackMapFix byte = 0x80 // up to 15 items + msgpackMap16 byte = 0xde // up to 2^16-1 items, followed by size in 2 bytes + msgpackMap32 byte = 0xdf // up to 2^32-1 items, followed by size in 4 bytes ) // safePayload provides a thread-safe wrapper around payload. @@ -152,29 +158,3 @@ func (sp *safePayload) protocol() float64 { // Protocol is immutable after creation - no lock needed return sp.p.protocol() } - -// traceChunk represents a list of spans with the same trace ID, -// i.e. a chunk of a trace -type traceChunk struct { - // the sampling priority of the trace - priority int32 `msg:"priority"` - - // the optional string origin ("lambda", "rum", etc.) of the trace chunk - origin string `msg:"origin,omitempty"` - - // a collection of key to value pairs common in all `spans` - attributes keyValueList `msg:"attributes,omitempty"` - - // a list of spans in this chunk - spans spanList `msg:"spans,omitempty"` - - // whether the trace only contains analyzed spans - // (not required by tracers and set by the agent) - droppedTrace bool `msg:"droppedTrace"` - - // the ID of the trace to which all spans in this chunk belong - traceID []byte `msg:"traceID"` - - // the optional string decision maker (previously span tag _dd.p.dm) - samplingMechanism string `msg:"samplingMechanism,omitempty"` -} diff --git a/ddtrace/tracer/payload_test.go b/ddtrace/tracer/payload_test.go index 4c840904b5..51e005d919 100644 --- a/ddtrace/tracer/payload_test.go +++ b/ddtrace/tracer/payload_test.go @@ -9,18 +9,22 @@ import ( "bytes" "fmt" "io" + "math" "strconv" "strings" "sync" "sync/atomic" "testing" + "github.com/DataDog/dd-trace-go/v2/internal/globalconfig" + "github.com/DataDog/dd-trace-go/v2/internal/version" "github.com/stretchr/testify/assert" "github.com/tinylib/msgp/msgp" ) var fixedTime = now() +// creates a simple span list with n spans func newSpanList(n int) spanList { itoa := map[int]string{0: "0", 1: "1", 2: "2", 3: "3", 4: "4", 5: "5"} list := make([]*Span, n) @@ -31,6 +35,23 @@ func newSpanList(n int) spanList { return list } +// creates a list of n spans, populated with SpanLinks, SpanEvents, and other fields +func newDetailedSpanList(n int) spanList { + itoa := map[int]string{0: "0", 1: "1", 2: "2", 3: "3", 4: "4", 5: "5"} + list := make([]*Span, n) + for i := 0; i < n; i++ { + list[i] = newBasicSpan("span.list." + itoa[i%5+1]) + list[i].start = fixedTime + list[i].service = "service." + itoa[i%5+1] + list[i].resource = "resource." + itoa[i%5+1] + list[i].error = int32(i % 2) + list[i].SetTag("tag."+itoa[i%5+1], "value."+itoa[i%5+1]) + // list[i].spanLinks = []SpanLink{{TraceID: 1, SpanID: 1}, {TraceID: 2, SpanID: 2}} + // list[i].spanEvents = []spanEvent{{Name: "span.event." + itoa[i%5+1]}} + } + return list +} + // TestPayloadIntegrity tests that whatever we push into the payload // allows us to read the same content as would have been encoded by // the codec. @@ -81,18 +102,19 @@ func TestPayloadV04Decode(t *testing.T) { // be decoded by the codec, and that it matches the original payload. func TestPayloadV1Decode(t *testing.T) { for _, n := range []int{10, 1 << 10} { - t.Run(strconv.Itoa(n), func(t *testing.T) { - assert := assert.New(t) - p := newPayloadV1() - - p.containerID = "containerID" - p.languageName = "languageName" - p.languageVersion = "languageVersion" - p.tracerVersion = "tracerVersion" - p.runtimeID = "runtimeID" - p.env = "env" - p.hostname = "hostname" - p.appVersion = "appVersion" + t.Run("simple"+strconv.Itoa(n), func(t *testing.T) { + var ( + assert = assert.New(t) + p = newPayloadV1() + ) + p.SetContainerID("containerID") + p.SetLanguageName("go") + p.SetLanguageVersion("1.25") + p.SetTracerVersion(version.Tag) + p.SetRuntimeID(globalconfig.RuntimeID()) + p.SetEnv("test") + p.SetHostname("hostname") + p.SetAppVersion("appVersion") for i := 0; i < n; i++ { _, _ = p.push(newSpanList(i%5 + 1)) @@ -102,9 +124,14 @@ func TestPayloadV1Decode(t *testing.T) { assert.NoError(err) got := newPayloadV1() - _, err = got.Decode(encoded) + buf := bytes.NewBuffer(encoded) + _, err = buf.WriteTo(got) assert.NoError(err) + o, err := got.decodeBuffer() + assert.NoError(err) + assert.Empty(o) + assert.Equal(p.fields, got.fields) assert.Equal(p.containerID, got.containerID) assert.Equal(p.languageName, got.languageName) assert.Equal(p.languageVersion, got.languageVersion) @@ -113,6 +140,76 @@ func TestPayloadV1Decode(t *testing.T) { assert.Equal(p.env, got.env) assert.Equal(p.hostname, got.hostname) assert.Equal(p.appVersion, got.appVersion) + assert.Equal(p.fields, got.fields) + }) + + t.Run("detailed"+strconv.Itoa(n), func(t *testing.T) { + var ( + assert = assert.New(t) + p = newPayloadV1() + ) + + for i := 0; i < n; i++ { + _, _ = p.push(newDetailedSpanList(i%5 + 1)) + } + encoded, err := io.ReadAll(p) + assert.NoError(err) + + got := newPayloadV1() + buf := bytes.NewBuffer(encoded) + _, err = buf.WriteTo(got) + assert.NoError(err) + + _, err = got.decodeBuffer() + assert.NoError(err) + assert.NotEmpty(got.attributes) + }) + } +} + +func TestPayloadV1EmbeddedStreamingStringTable(t *testing.T) { + p := newPayloadV1() + p.SetHostname("production") + p.SetEnv("production") + p.SetLanguageName("go") + + assert := assert.New(t) + encoded, err := io.ReadAll(p) + assert.NoError(err) + + got := newPayloadV1() + buf := bytes.NewBuffer(encoded) + _, err = buf.WriteTo(got) + assert.NoError(err) + + o, err := got.decodeBuffer() + assert.NoError(err) + assert.Empty(o) + assert.Equal(p.languageName, got.languageName) + assert.Equal(p.hostname, got.hostname) + assert.Equal(p.env, got.env) +} + +func TestPayloadV1UpdateHeader(t *testing.T) { + testCases := []uint32{ // Number of items + 15, + math.MaxUint16, + math.MaxUint32, + } + for _, tc := range testCases { + t.Run(fmt.Sprintf("n=%d", tc), func(t *testing.T) { + var ( + p = payloadV1{ + fields: tc, + header: make([]byte, 8), + } + expected []byte + ) + expected = msgp.AppendMapHeader(expected, tc) + p.updateHeader() + if got := p.header[p.readOff:]; !bytes.Equal(expected, got) { + t.Fatalf("expected %+v, got %+v", expected, got) + } }) } } diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index a8044a06e8..02c5670945 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -11,7 +11,6 @@ import ( "fmt" "sync/atomic" - "github.com/DataDog/dd-trace-go/v2/ddtrace/ext" "github.com/tinylib/msgp/msgp" ) @@ -29,40 +28,39 @@ import ( // payloadV1 for re-use requires the transport to wait for the HTTP package // Close the request body before attempting to re-use it again! type payloadV1 struct { - // array of strings referenced in this tracer payload, its chunks and spans - // stringTable holds references from a string value to an index. - // the 0th position in the stringTable should always be the empty string. - strings *stringTable `msgp:"strings"` + // bm keeps track of which fields have been set in the payload + // bits 1-11 are used for field IDs 1-11. Bit 0 is unused. + bm bitmap // the string ID of the container where the tracer is running - containerID string `msgp:"containerID"` + containerID string // 2 // the string language name of the tracer - languageName string `msgp:"languageName"` + languageName string // 3 // the string language version of the tracer - languageVersion string `msgp:"languageVersion"` + languageVersion string // 4 // the string version of the tracer - tracerVersion string `msgp:"tracerVersion"` + tracerVersion string // 5 // the V4 string UUID representation of a tracer session - runtimeID string `msgp:"runtimeID"` + runtimeID string // 6 // the optional `env` string tag that set with the tracer - env string `msgp:"env,omitempty"` + env string // 7 // the optional string hostname of where the tracer is running - hostname string `msgp:"hostname,omitempty"` + hostname string // 8 // the optional string `version` tag for the application set in the tracer - appVersion string `msgp:"appVersion,omitempty"` + appVersion string // 9 // a collection of key to value pairs common in all `chunks` - attributes keyValueList `msgp:"attributes,omitempty"` + attributes map[string]anyValue // 10 // a list of trace `chunks` - chunks []traceChunk `msgp:"chunks,omitempty"` + chunks []traceChunk // 11 // protocolVersion specifies the trace protocol to use. protocolVersion float64 @@ -72,152 +70,92 @@ type payloadV1 struct { // and the number of items contained in the stream. header []byte - // off specifies the current read position on the header. - off int + // readOff specifies the current read position on the header. + readOff int - // count specifies the number of items in the stream. + // writeOff specifies the current read position on the header. + writeOff int + + // count specifies the number of items (traceChunks) in the stream. count uint32 + // fields specifies the number of fields in the payload. + fields uint32 + // buf holds the sequence of msgpack-encoded items. - buf bytes.Buffer + buf []byte // reader is used for reading the contents of buf. reader *bytes.Reader } -type stringTable struct { - strings []string // list of strings - indices map[string]uint32 // map strings to their indices - nextIndex uint32 // last index of the stringTable -} - -// AnyValue is a representation of the `any` value. It can take the following types: -// - uint32 -// - bool -// - float64 -// - int64 -// - uint8 -// intValue(5) - 0x405 (4 indicates this is an int AnyType, then 5 is encoded using positive fixed int format) -// stringValue(“a”) - 0x1a161 (1 indicates this is a string, then “a” is encoded using fixstr 0xa161) -// stringValue(2) - 0x102 (1 indicates this is a string, then a positive fixed int of 2 refers the 2nd index of the string table) -type anyValue struct { - valueType int - value interface{} -} - -const ( - StringValueType = iota + 1 // string or uint -- 1 - BoolValueType // boolean -- 2 - FloatValueType // float64 -- 3 - IntValueType // int64 -- 4 - BytesValueType // []uint8 -- 5 - ArrayValueType // []AnyValue -- 6 - keyValueListType // []keyValue -- 7 -) - -type arrayValue []anyValue - -// keys in a keyValue can either be a string or a uint32 index -// isString is true when the key is a string value, and false when the key is a uint32 index -type streamingKey struct { - isString bool - stringValue string - idx uint32 -} - -// keyValue is made up of the key and an AnyValue (the type of the value and the value itself) -// The key is either a uint32 index into the string table or a string value. -type keyValue struct { - key streamingKey - value anyValue -} - -type keyValueList []keyValue - // newPayloadV1 returns a ready to use payloadV1. func newPayloadV1() *payloadV1 { return &payloadV1{ protocolVersion: traceProtocolV1, - attributes: keyValueList{}, + attributes: make(map[string]anyValue), chunks: make([]traceChunk, 0), - strings: newStringTable(), - header: make([]byte, 8), - off: 8, + readOff: 8, + writeOff: 0, } } -func newStringTable() *stringTable { - return &stringTable{ - strings: []string{""}, - indices: map[string]uint32{"": 0}, - nextIndex: 1, - } -} - -func (s *stringTable) Add(str string) { - if _, ok := s.indices[str]; ok { - return - } - s.indices[str] = s.nextIndex - s.strings = append(s.strings, str) - s.nextIndex += 1 -} - -// push pushes a new item into the stream. +// push pushes a new item (a traceChunk)into the payload. func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { // We need to hydrate the payload with everything we get from the spans. - // Conceptually, our `t []*Span` corresponds to one `traceChunk`. - origin, priority := "", 0 + // Conceptually, our `t spanList` corresponds to one `traceChunk`. + attributes := map[string]anyValue{} for _, span := range t { if span == nil { continue } - if p, ok := span.Context().SamplingPriority(); ok { - origin = span.Context().origin - priority = p - break + for k, v := range span.meta { + av := anyValue{valueType: StringValueType, value: v} + attributes[k] = av + p.attributes[k] = av } - } - - kv := keyValueList{ - {key: streamingKey{isString: false, idx: 2}, value: anyValue{valueType: StringValueType, value: p.containerID}}, // containerID - {key: streamingKey{isString: false, idx: 3}, value: anyValue{valueType: StringValueType, value: p.languageName}}, // languageName - {key: streamingKey{isString: false, idx: 4}, value: anyValue{valueType: StringValueType, value: p.languageVersion}}, // languageVersion - {key: streamingKey{isString: false, idx: 5}, value: anyValue{valueType: StringValueType, value: p.tracerVersion}}, // tracerVersion - {key: streamingKey{isString: false, idx: 6}, value: anyValue{valueType: StringValueType, value: p.runtimeID}}, // runtimeID - {key: streamingKey{isString: false, idx: 7}, value: anyValue{valueType: StringValueType, value: p.env}}, // env - {key: streamingKey{isString: false, idx: 8}, value: anyValue{valueType: StringValueType, value: p.hostname}}, // hostname - {key: streamingKey{isString: false, idx: 9}, value: anyValue{valueType: StringValueType, value: p.appVersion}}, // appVersion + for k, v := range span.metrics { + av := anyValue{valueType: FloatValueType, value: v} + attributes[k] = av + p.attributes[k] = av + } + // for k, v := range span.metaStruct { + // attributes = append(attributes, keyValue{key: k, value: anyValue{valueType: keyValueListType, value: v}}) + // } } tc := traceChunk{ - priority: int32(priority), - origin: origin, - attributes: keyValueList{}, - spans: t, - traceID: t[0].Context().traceID[:], + spans: t, + traceID: t[0].Context().traceID[:], } - p.chunks = append(p.chunks, tc) - wr := msgp.NewWriter(&p.buf) - err = tc.EncodeMsg(wr, p) - if err != nil { - return payloadStats{}, err + // if there are attributes available, set them in our bitmap and increment + // the number of fields. + if len(attributes) > 0 { + tc.attributes = attributes + p.bm.set(10) + p.fields += 1 } - - // once we've encoded the spans, we can encode the attributes - kv = append(kv, keyValue{key: streamingKey{isString: false, idx: 10}, value: anyValue{valueType: keyValueListType, value: p.attributes}}) // attributes - err = kv.EncodeMsg(wr, p) - if err == nil { - err = wr.Flush() - } - + p.chunks = append(p.chunks, tc) p.recordItem() return p.stats(), err } +// grows the buffer to fit n more bytes. Follows the internal Go standard +// for growing slices (https://github.com/golang/go/blob/master/src/runtime/slice.go#L289) func (p *payloadV1) grow(n int) { - p.buf.Grow(n) + cap := cap(p.buf) + newLen := len(p.buf) + n + threshold := 256 + for { + cap += (cap + 3*threshold) >> 2 + if cap >= newLen { + break + } + } + newBuffer := make([]byte, cap) + copy(newBuffer, p.buf) + p.buf = newBuffer } func (p *payloadV1) reset() { @@ -228,13 +166,14 @@ func (p *payloadV1) reset() { } func (p *payloadV1) clear() { - p.buf = bytes.Buffer{} + p.bm = 0 + p.buf = p.buf[:] p.reader = nil } func (p *payloadV1) recordItem() { atomic.AddUint32(&p.count, 1) - p.updateHeader() + // p.updateHeader() TODO(hannahkm): figure out } func (p *payloadV1) stats() payloadStats { @@ -245,7 +184,7 @@ func (p *payloadV1) stats() payloadStats { } func (p *payloadV1) size() int { - return p.buf.Len() + len(p.header) - p.off + return len(p.buf) + len(p.header) - p.readOff } func (p *payloadV1) itemCount() int { @@ -257,169 +196,193 @@ func (p *payloadV1) protocol() float64 { } func (p *payloadV1) updateHeader() { - n := uint64(atomic.LoadUint32(&p.count)) + n := uint64(p.fields) switch { case n <= 15: - p.header[7] = msgpackArrayFix + byte(n) - p.off = 7 + p.header[7] = msgpackMapFix + byte(n) + p.readOff = 7 case n <= 1<<16-1: binary.BigEndian.PutUint64(p.header, n) // writes 2 bytes - p.header[5] = msgpackArray16 - p.off = 5 + p.header[5] = msgpackMap16 + p.readOff = 5 default: // n <= 1<<32-1 binary.BigEndian.PutUint64(p.header, n) // writes 4 bytes - p.header[3] = msgpackArray32 - p.off = 3 + p.header[3] = msgpackMap32 + p.readOff = 3 } } -// Close implements io.Closer func (p *payloadV1) Close() error { + p.clear() return nil } -// Write implements io.Writer. It writes data directly to the buffer. -func (p *payloadV1) Write(data []byte) (n int, err error) { - return p.buf.Write(data) +func (p *payloadV1) Write(b []byte) (int, error) { + p.buf = append(p.buf, b...) + return len(b), nil } // Read implements io.Reader. It reads from the msgpack-encoded stream. func (p *payloadV1) Read(b []byte) (n int, err error) { - if p.off < len(p.header) { + if len(p.header) == 0 { + p.header = make([]byte, 8) + p.updateHeader() + } + if p.readOff < len(p.header) { // reading header - n = copy(b, p.header[p.off:]) - p.off += n + n = copy(b, p.header[p.readOff:]) + p.readOff += n return n, nil } + if len(p.buf) == 0 { + p.encode() + } if p.reader == nil { - p.reader = bytes.NewReader(p.buf.Bytes()) + p.reader = bytes.NewReader(p.buf) } return p.reader.Read(b) } -// Encode the anyValue -func (a *anyValue) EncodeMsg(e *msgp.Writer, p *payloadV1) error { - switch a.valueType { - case StringValueType: - e.WriteInt32(StringValueType) - v, err := p.encodeString(a.value.(string)) - if err != nil { - return err - } - if v.isString { - return e.WriteString(v.stringValue) +// encode writes existing payload fields into the buffer in msgp format. +func (p *payloadV1) encode() { + st := newStringTable() + p.encodeField(p.bm, 2, anyValue{valueType: StringValueType, value: p.containerID}, st) + p.encodeField(p.bm, 3, anyValue{valueType: StringValueType, value: p.languageName}, st) + p.encodeField(p.bm, 4, anyValue{valueType: StringValueType, value: p.languageVersion}, st) + p.encodeField(p.bm, 5, anyValue{valueType: StringValueType, value: p.tracerVersion}, st) + p.encodeField(p.bm, 6, anyValue{valueType: StringValueType, value: p.runtimeID}, st) + p.encodeField(p.bm, 7, anyValue{valueType: StringValueType, value: p.env}, st) + p.encodeField(p.bm, 8, anyValue{valueType: StringValueType, value: p.hostname}, st) + p.encodeField(p.bm, 9, anyValue{valueType: StringValueType, value: p.appVersion}, st) + + if len(p.attributes) > 0 { + p.encodeAttributes(10, p.attributes, st) + } + + if len(p.chunks) > 0 { + p.encodeTraceChunks(11, p.chunks, st) + } +} + +// TODO(hannahkm): is this the best way to go about encoding fields? +func (p *payloadV1) encodeField(bm bitmap, fieldID int, a anyValue, st *stringTable) { + if !bm.contains(uint32(fieldID)) { + return + } + p.buf = msgp.AppendUint32(p.buf, uint32(fieldID)) // msgp key + // p.buf = msgp.AppendInt32(p.buf, int32(a.valueType)) // value type + if a.valueType == StringValueType { + value := a.value.(string) + // encode msgp value, either by pulling from string table or writing it directly + if idx, ok := st.Get(value); ok { + p.buf = idx.encode(p.buf) + } else { + s := stringValue(value) + p.buf = s.encode(p.buf) + st.Add(value) } - return e.WriteUint32(v.idx) + return + } + switch a.valueType { case BoolValueType: - e.WriteInt32(BoolValueType) - return e.WriteBool(a.value.(bool)) + p.buf = msgp.AppendBool(p.buf, a.value.(bool)) case FloatValueType: - e.WriteInt32(FloatValueType) - return e.WriteFloat64(a.value.(float64)) + p.buf = msgp.AppendFloat64(p.buf, a.value.(float64)) case IntValueType: - e.WriteInt32(IntValueType) - return e.WriteInt64(a.value.(int64)) + p.buf = msgp.AppendInt64(p.buf, a.value.(int64)) case BytesValueType: - e.WriteInt32(BytesValueType) - return e.WriteBytes(a.value.([]byte)) + p.buf = msgp.AppendBytes(p.buf, a.value.([]byte)) case ArrayValueType: - e.WriteInt32(ArrayValueType) - return a.value.(arrayValue).EncodeMsg(e, p) - case keyValueListType: - e.WriteInt32(keyValueListType) - return a.value.(keyValueList).EncodeMsg(e, p) - default: - return fmt.Errorf("invalid value type: %d", a.valueType) + p.buf = msgp.AppendArrayHeader(p.buf, uint32(len(a.value.(arrayValue)))) + for _, v := range a.value.(arrayValue) { + v.encode(p.buf) + } } } -func (av arrayValue) EncodeMsg(e *msgp.Writer, p *payloadV1) error { - err := e.WriteArrayHeader(uint32(len(av))) - if err != nil { - return err +func (p *payloadV1) encodeAttributes(fieldID int, kv map[string]anyValue, st *stringTable) error { + if !p.bm.contains(uint32(fieldID)) || len(kv) == 0 { + return nil } - for _, value := range av { - if err := value.EncodeMsg(e, p); err != nil { - return err + + p.buf = msgp.AppendUint32(p.buf, uint32(fieldID)) // msgp key + p.buf = msgp.AppendMapHeader(p.buf, uint32(len(kv))) // number of item pairs in map + for k, v := range kv { + // encode msgp key + if idx, ok := st.Get(string(k)); ok { + p.buf = idx.encode(p.buf) + } else { + p.buf = stringValue(k).encode(p.buf) + st.Add(string(k)) } - } - return nil -} -func (k keyValue) EncodeMsg(e *msgp.Writer, p *payloadV1) error { - var err error - if k.key.isString { - err = e.WriteString(k.key.stringValue) - } else { - err = e.WriteUint32(k.key.idx) - } - if err != nil { - return err - } - err = k.value.EncodeMsg(e, p) - if err != nil { - return err + // encode value + p.buf = v.encode(p.buf) } return nil } -func (kv keyValueList) EncodeMsg(e *msgp.Writer, p *payloadV1) error { - err := e.WriteMapHeader(uint32(len(kv))) - if err != nil { - return err - } - for _, k := range kv { - err = k.EncodeMsg(e, p) - if err != nil { - return err - } +func (p *payloadV1) encodeTraceChunks(fieldID int, tc []traceChunk, st *stringTable) error { + if len(tc) == 0 { + return nil } - return nil -} -func (t *traceChunk) EncodeMsg(e *msgp.Writer, p *payloadV1) error { - e.WriteInt32(11) // write msgp index for `chunks` + p.buf = msgp.AppendUint32(p.buf, uint32(fieldID)) // msgp key + p.buf = msgp.AppendArrayHeader(p.buf, uint32(len(tc))) // number of chunks + for _, chunk := range tc { + p.buf = msgp.AppendMapHeader(p.buf, uint32(chunk.fields)) // number of item pairs in map - kv := keyValueList{ - {key: streamingKey{isString: false, idx: 1}, value: anyValue{valueType: IntValueType, value: int64(t.priority)}}, // priority - {key: streamingKey{isString: false, idx: 2}, value: anyValue{valueType: StringValueType, value: t.origin}}, // origin - {key: streamingKey{isString: false, idx: 5}, value: anyValue{valueType: BoolValueType, value: t.droppedTrace}}, // droppedTrace - {key: streamingKey{isString: false, idx: 6}, value: anyValue{valueType: BytesValueType, value: t.traceID}}, // traceID - {key: streamingKey{isString: false, idx: 7}, value: anyValue{valueType: StringValueType, value: t.samplingMechanism}}, // samplingMechanism - } + // priority + if chunk.bm.contains(1) { + p.buf = msgp.AppendUint32(p.buf, 1) // field ID + p.buf = msgp.AppendInt32(p.buf, chunk.priority) + } - attr := keyValueList{} - for k, v := range t.attributes { - attr = append(attr, keyValue{key: streamingKey{isString: false, idx: uint32(k)}, value: anyValue{valueType: getAnyValueType(v), value: v}}) - } - kv = append(kv, keyValue{key: streamingKey{isString: false, idx: 3}, value: anyValue{valueType: keyValueListType, value: attr}}) // attributes + // origin + if chunk.bm.contains(2) { + p.buf = msgp.AppendUint32(p.buf, 2) // field ID + // encode msgp value, either by pulling from string table or writing it directly + if idx, ok := st.Get(chunk.origin); ok { + p.buf = idx.encode(p.buf) + } else { + s := stringValue(chunk.origin) + p.buf = s.encode(p.buf) + st.Add(chunk.origin) + } + } - err := kv.EncodeMsg(e, p) - if err != nil { - return err - } + // attributes + if chunk.bm.contains(3) { + p.encodeAttributes(3, chunk.attributes, st) + } - return EncodeSpanList(t.spans, e, p) -} + // spans + if chunk.bm.contains(4) { + p.encodeSpans(4, chunk.spans, st) + } -func EncodeSpanList(s spanList, e *msgp.Writer, p *payloadV1) error { - e.WriteInt32(4) // write msgp index for `spans` + // droppedTrace + if chunk.bm.contains(5) { + p.buf = msgp.AppendUint32(p.buf, 5) // field ID + p.buf = msgp.AppendBool(p.buf, chunk.droppedTrace) + } - err := e.WriteArrayHeader(uint32(len(s))) - if err != nil { - return msgp.WrapError(err) - } + // traceID + if chunk.bm.contains(6) { + p.buf = msgp.AppendUint32(p.buf, 6) // field ID + p.buf = msgp.AppendBytes(p.buf, chunk.traceID) + } - for _, span := range s { - if span == nil { - err := e.WriteNil() - if err != nil { - return err - } - } else { - err := encodeSpan(span, e, p) - if err != nil { - return msgp.WrapError(err, span) + // samplingMechanism + if chunk.bm.contains(7) { + p.buf = msgp.AppendUint32(p.buf, 7) // field ID + // encode msgp value, either by pulling from string table or writing it directly + if idx, ok := st.Get(chunk.samplingMechanism); ok { + p.buf = idx.encode(p.buf) + } else { + s := stringValue(chunk.samplingMechanism) + p.buf = s.encode(p.buf) + st.Add(chunk.samplingMechanism) } } } @@ -427,325 +390,424 @@ func EncodeSpanList(s spanList, e *msgp.Writer, p *payloadV1) error { return nil } -// Custom encoding for spans under the v1 trace protocol. -// The encoding of attributes is the combination of the meta, metrics, and metaStruct fields of the v0.4 protocol. -func encodeSpan(s *Span, e *msgp.Writer, p *payloadV1) error { - kv := keyValueList{ - {key: streamingKey{isString: false, idx: 1}, value: anyValue{valueType: StringValueType, value: s.service}}, // service - {key: streamingKey{isString: false, idx: 2}, value: anyValue{valueType: StringValueType, value: s.name}}, // name - {key: streamingKey{isString: false, idx: 3}, value: anyValue{valueType: StringValueType, value: s.resource}}, // resource - {key: streamingKey{isString: false, idx: 4}, value: anyValue{valueType: IntValueType, value: int64(s.spanID)}}, // spanID - {key: streamingKey{isString: false, idx: 5}, value: anyValue{valueType: IntValueType, value: int64(s.parentID)}}, // parentID - {key: streamingKey{isString: false, idx: 6}, value: anyValue{valueType: IntValueType, value: int64(s.start)}}, // start - {key: streamingKey{isString: false, idx: 7}, value: anyValue{valueType: IntValueType, value: int64(s.duration)}}, // duration - {key: streamingKey{isString: false, idx: 8}, value: anyValue{valueType: BoolValueType, value: (s.error != 0)}}, // error - true if span has error - {key: streamingKey{isString: false, idx: 10}, value: anyValue{valueType: StringValueType, value: s.spanType}}, // type - {key: streamingKey{isString: false, idx: 15}, value: anyValue{valueType: StringValueType, value: s.integration}}, // component +func (p *payloadV1) encodeSpans(fieldID int, spans spanList, st *stringTable) error { + if len(spans) == 0 || !p.bm.contains(uint32(fieldID)) { + return nil } - // encode meta attributes - attr := keyValueList{} - for k, v := range s.meta { - idx, err := p.encodeString(k) - if err != nil { - idx = streamingKey{isString: true, stringValue: k} - } - attr = append(attr, keyValue{key: idx, value: anyValue{valueType: StringValueType, value: v}}) - } + p.buf = msgp.AppendUint32(p.buf, uint32(fieldID)) // msgp key + p.buf = msgp.AppendArrayHeader(p.buf, uint32(len(spans))) // number of spans - // encode metric attributes - for k, v := range s.metrics { - idx, err := p.encodeString(k) - if err != nil { - idx = streamingKey{isString: true, stringValue: k} + for _, span := range spans { + if span == nil { + continue + } + // TODO(hannahkm): how do we get the number of set fields efficiently? + // TODO(hannahkm): might need to change the bitmap value in the calls below + p.encodeField(p.bm, 1, anyValue{valueType: StringValueType, value: span.service}, st) + p.encodeField(p.bm, 2, anyValue{valueType: StringValueType, value: span.name}, st) + p.encodeField(p.bm, 3, anyValue{valueType: StringValueType, value: span.resource}, st) + p.encodeField(p.bm, 4, anyValue{valueType: IntValueType, value: span.spanID}, st) + p.encodeField(p.bm, 5, anyValue{valueType: IntValueType, value: span.parentID}, st) + p.encodeField(p.bm, 6, anyValue{valueType: IntValueType, value: span.start}, st) + p.encodeField(p.bm, 7, anyValue{valueType: IntValueType, value: span.duration}, st) + if span.error != 0 { + p.encodeField(p.bm, 8, anyValue{valueType: BoolValueType, value: true}, st) + } else { + p.encodeField(p.bm, 8, anyValue{valueType: BoolValueType, value: false}, st) } - attr = append(attr, keyValue{key: idx, value: anyValue{valueType: FloatValueType, value: v}}) + p.encodeField(p.bm, 10, anyValue{valueType: StringValueType, value: span.spanType}, st) + p.encodeSpanLinks(11, span.spanLinks, st) + p.encodeSpanEvents(12, span.spanEvents, st) + p.encodeField(p.bm, 15, anyValue{valueType: StringValueType, value: span.integration}, st) + + // TODO(hannahkm): add attributes, env, version } + return nil +} - // encode metaStruct attributes - for k, v := range s.metaStruct { - idx, err := p.encodeString(k) - if err != nil { - idx = streamingKey{isString: true, stringValue: k} - } - attr = append(attr, keyValue{key: idx, value: anyValue{valueType: getAnyValueType(v), value: v}}) +func (p *payloadV1) encodeSpanLinks(fieldID int, spanLinks []SpanLink, st *stringTable) error { + if len(spanLinks) == 0 || !p.bm.contains(uint32(fieldID)) { + return nil } + p.buf = msgp.AppendUint32(p.buf, uint32(fieldID)) // msgp key + p.buf = msgp.AppendArrayHeader(p.buf, uint32(len(spanLinks))) // number of span links - kv = append(kv, keyValue{key: streamingKey{isString: false, idx: 9}, value: anyValue{valueType: keyValueListType, value: attr}}) // attributes + for _, link := range spanLinks { + // TODO(hannahkm): how do we get the number of set fields + // TODO(hannahkm): might need to change the bitmap value in the calls below + p.encodeField(p.bm, 1, anyValue{valueType: BytesValueType, value: link.TraceID}, st) + p.encodeField(p.bm, 2, anyValue{valueType: IntValueType, value: link.SpanID}, st) + p.encodeField(p.bm, 4, anyValue{valueType: StringValueType, value: link.Tracestate}, st) + p.encodeField(p.bm, 5, anyValue{valueType: StringValueType, value: link.Flags}, st) - env, ok := s.meta["env"] - if ok { - kv = append(kv, keyValue{key: streamingKey{isString: false, idx: 13}, value: anyValue{valueType: StringValueType, value: env}}) // env - } - version, ok := s.meta["version"] - if ok { - kv = append(kv, keyValue{key: streamingKey{isString: false, idx: 14}, value: anyValue{valueType: StringValueType, value: version}}) // version + // TODO(hannahkm): add attributes } + return nil +} - err := kv.EncodeMsg(e, p) - if err != nil { - return err +func (p *payloadV1) encodeSpanEvents(fieldID int, spanEvents []spanEvent, st *stringTable) error { + if len(spanEvents) == 0 || !p.bm.contains(uint32(fieldID)) { + return nil } + p.buf = msgp.AppendUint32(p.buf, uint32(fieldID)) // msgp key + p.buf = msgp.AppendArrayHeader(p.buf, uint32(len(spanEvents))) // number of span events - // spanLinks - err = encodeSpanLinks(s.spanLinks, e, p) - if err != nil { - return err + for _, event := range spanEvents { + // TODO(hannahkm): how do we get the number of set fields + // TODO(hannahkm): might need to change the bitmap value in the calls below + p.encodeField(p.bm, 1, anyValue{valueType: IntValueType, value: event.TimeUnixNano}, st) + p.encodeField(p.bm, 2, anyValue{valueType: StringValueType, value: event.Name}, st) + // TODO(hannahkm): add attributes } + return nil +} - // spanEvents - return encodeSpanEvents(s.spanEvents, e, p) +// Getters for payloadV1 fields +func (p *payloadV1) GetContainerID() string { return p.containerID } +func (p *payloadV1) GetLanguageName() string { return p.languageName } +func (p *payloadV1) GetLanguageVersion() string { return p.languageVersion } +func (p *payloadV1) GetTracerVersion() string { return p.tracerVersion } +func (p *payloadV1) GetRuntimeID() string { return p.runtimeID } +func (p *payloadV1) GetEnv() string { return p.env } +func (p *payloadV1) GetHostname() string { return p.hostname } +func (p *payloadV1) GetAppVersion() string { return p.appVersion } +func (p *payloadV1) GetAttributes() map[string]anyValue { return p.attributes } + +func (p *payloadV1) SetContainerID(v string) { + p.containerID = v + p.bm.set(2) + p.fields += 1 } -// encodeString and decodeString handles encoding a string to the payload's string table. -// When writing a string: -// - use its index in the string table if it exists -// - otherwise, write the string into the message, then add the string at the next index -// Returns the index of the string in the string table, and an error if there is one -func (p *payloadV1) encodeString(s string) (streamingKey, error) { - sTable := p.strings - idx, ok := sTable.indices[s] - // if the string already exists in the table, use its index - if ok { - return streamingKey{isString: false, idx: idx}, nil - } +func (p *payloadV1) SetLanguageName(v string) { + p.languageName = v + p.bm.set(3) + p.fields += 1 +} - // else, write the string into the table at the next index - // return an error to indicate that the string should be written to the msgp message - sTable.Add(s) - return streamingKey{isString: true, stringValue: s}, nil +func (p *payloadV1) SetLanguageVersion(v string) { + p.languageVersion = v + p.bm.set(4) + p.fields += 1 } -// encodeSpanLinks encodes the span links into a msgp.Writer -// Span links are represented as an array of fixmaps (keyValueList) -func encodeSpanLinks(sl []SpanLink, e *msgp.Writer, p *payloadV1) error { - err := e.WriteInt32(11) // spanLinks - if err != nil { - return err - } +func (p *payloadV1) SetTracerVersion(v string) { + p.tracerVersion = v + p.bm.set(5) + p.fields += 1 +} - // write the number of span links - err = e.WriteArrayHeader(uint32(len(sl))) +func (p *payloadV1) SetRuntimeID(v string) { + p.runtimeID = v + p.bm.set(6) + p.fields += 1 +} + +func (p *payloadV1) SetEnv(v string) { + p.env = v + p.bm.set(7) + p.fields += 1 +} + +func (p *payloadV1) SetHostname(v string) { + p.hostname = v + p.bm.set(8) + p.fields += 1 +} + +func (p *payloadV1) SetAppVersion(v string) { + p.appVersion = v + p.bm.set(9) + p.fields += 1 +} + +// decodeBuffer takes the buffer from the payload, decodes it, and populates the fields +// according to the msgpack-encoded byte stream. +func (p *payloadV1) decodeBuffer() ([]byte, error) { + numFields, o, err := msgp.ReadMapHeaderBytes(p.buf) if err != nil { - return err + return o, err } + p.buf = o + p.fields = numFields + p.header = make([]byte, 8) + p.updateHeader() - // represent each span link as a fixmap (keyValueList) and add it to an array - kv := arrayValue{} - for _, s := range sl { - slKeyValues := keyValueList{ - {key: streamingKey{isString: false, idx: 1}, value: anyValue{valueType: IntValueType, value: int64(s.TraceID)}}, // traceID - {key: streamingKey{isString: false, idx: 2}, value: anyValue{valueType: IntValueType, value: int64(s.SpanID)}}, // spanID - {key: streamingKey{isString: false, idx: 4}, value: anyValue{valueType: StringValueType, value: s.Tracestate}}, // tracestate - {key: streamingKey{isString: false, idx: 5}, value: anyValue{valueType: IntValueType, value: int64(s.Flags)}}, // flags + st := newStringTable() + for { + // read msgp field ID + var idx uint32 + idx, o, err = msgp.ReadUint32Bytes(o) + if err != nil { + break } - attr := keyValueList{} - // attributes - for k, v := range s.Attributes { - idx, err := p.encodeString(k) + // handle attributes + if idx == 10 { + p.attributes, o, err = DecodeKeyValueList(o, st) if err != nil { - idx = streamingKey{isString: true, stringValue: k} + break } - attr = append(attr, keyValue{key: idx, value: anyValue{valueType: getAnyValueType(v), value: v}}) + continue } - slKeyValues = append(slKeyValues, keyValue{key: streamingKey{isString: false, idx: 3}, value: anyValue{valueType: ArrayValueType, value: attr}}) // attributes - kv = append(kv, anyValue{valueType: keyValueListType, value: slKeyValues}) - } - for _, v := range kv { - err := v.EncodeMsg(e, p) - if err != nil { - return err + // handle trace chunks + if idx == 11 { + // p.chunks, o, err = DecodeTraceChunks(o, st) + // if err != nil { + // break + // } + continue } - } - return nil -} - -// encodeSpanEvents encodes the span events into a msgp.Writer -// Span events are represented as an array of fixmaps (keyValueList) -func encodeSpanEvents(se []spanEvent, e *msgp.Writer, p *payloadV1) error { - err := e.WriteInt32(12) // spanEvents - if err != nil { - return err - } - - // write the number of span events - err = e.WriteArrayHeader(uint32(len(se))) - if err != nil { - return err - } - // represent each span event as a fixmap (keyValueList) and add it to an array - kv := arrayValue{} - for _, s := range se { - slKeyValues := keyValueList{ - {key: streamingKey{isString: false, idx: 1}, value: anyValue{valueType: IntValueType, value: int64(s.TimeUnixNano)}}, // time - {key: streamingKey{isString: false, idx: 2}, value: anyValue{valueType: StringValueType, value: s.Name}}, // name + // read msgp string value + var value string + var ok bool + value, o, ok = st.Read(o) + if !ok { + err = fmt.Errorf("unable to read string value of field %d", idx) + break } - attr := keyValueList{} - // attributes - for k, v := range s.Attributes { - idx, err := p.encodeString(k) - if err != nil { - idx = streamingKey{isString: true, stringValue: k} - } - attr = append(attr, keyValue{key: idx, value: anyValue{valueType: getAnyValueType(v), value: v}}) + switch idx { + case 2: + p.containerID = value + case 3: + p.languageName = value + case 4: + p.languageVersion = value + case 5: + p.tracerVersion = value + case 6: + p.runtimeID = value + case 7: + p.env = value + case 8: + p.hostname = value + case 9: + p.appVersion = value + default: + err = fmt.Errorf("unexpected field ID %d", idx) + } + if len(o) == 0 || err != nil { + break } - slKeyValues = append(slKeyValues, keyValue{key: streamingKey{isString: false, idx: 3}, value: anyValue{valueType: ArrayValueType, value: attr}}) // attributes - kv = append(kv, anyValue{valueType: keyValueListType, value: slKeyValues}) } + return o, err +} - for _, v := range kv { - err := v.EncodeMsg(e, p) - if err != nil { - return err +// AnyValue is a representation of the `any` value. It can take the following types: +// - uint32 +// - bool +// - float64 +// - int64 +// - uint8 +// intValue(5) - 0x405 (4 indicates this is an int AnyType, then 5 is encoded using positive fixed int format) +// stringValue(“a”) - 0x1a161 (1 indicates this is a string, then “a” is encoded using fixstr 0xa161) +// stringValue(2) - 0x102 (1 indicates this is a string, then a positive fixed int of 2 refers the 2nd index of the string table) +type anyValue struct { + valueType int + value interface{} +} + +const ( + StringValueType = iota + 1 // string or uint -- 1 + BoolValueType // boolean -- 2 + FloatValueType // float64 -- 3 + IntValueType // int64 -- 4 + BytesValueType // []uint8 -- 5 + ArrayValueType // []AnyValue -- 6 + keyValueListType // []keyValue -- 7 +) + +func (a anyValue) encode(buf []byte) []byte { + buf = msgp.AppendInt32(buf, int32(a.valueType)) + switch a.valueType { + case StringValueType: + buf = a.value.(stringValue).encode(buf) + case BoolValueType: + buf = msgp.AppendBool(buf, a.value.(bool)) + case FloatValueType: + buf = msgp.AppendFloat64(buf, a.value.(float64)) + case IntValueType: + buf = msgp.AppendInt64(buf, a.value.(int64)) + case BytesValueType: + buf = msgp.AppendBytes(buf, a.value.([]byte)) + case ArrayValueType: + buf = msgp.AppendArrayHeader(buf, uint32(len(a.value.(arrayValue)))) + for _, v := range a.value.(arrayValue) { + v.encode(buf) } } - return nil + return buf } -func getAnyValueType(v any) int { - switch v.(type) { - case string: - return StringValueType - case bool: - return BoolValueType - case float64: - return FloatValueType - case float32: - return FloatValueType - case []byte: - return BytesValueType +type arrayValue []anyValue + +// keeps track of which fields have been set in the payload, with a +// 1 for represented fields and 0 for unset fields. +type bitmap int16 + +func (b *bitmap) set(bit uint32) { + if bit >= 16 { + return } - return IntValueType + *b |= 1 << bit } -func (p *payloadV1) Decode(b []byte) ([]byte, error) { - if p.strings == nil { - p.strings = newStringTable() +func (b bitmap) contains(bit uint32) bool { + if bit >= 16 { + return false } + return b&(1< 0 { - fields-- - - f, o, err := msgp.ReadInt32Bytes(o) - if err != nil { - return o, err - } - - switch f { - // we don't care for the string table, so we don't decode it - case 2: // containerID - p.containerID, o, err = DecodeStreamingString(o, p.strings) - if err != nil { - return o, err - } - - case 3: // languageName - p.languageName, o, err = DecodeStreamingString(o, p.strings) - if err != nil { - return o, err - } - - case 4: // languageVersion - p.languageVersion, o, err = DecodeStreamingString(o, p.strings) - if err != nil { - return o, err - } +// an encodable and decodable string value +type stringValue string - case 5: // tracerVersion - p.tracerVersion, o, err = DecodeStreamingString(o, p.strings) - if err != nil { - return o, err - } +func (s stringValue) encode(buf []byte) []byte { + return msgp.AppendString(buf, string(s)) +} - case 6: // runtimeID - p.runtimeID, o, err = DecodeStreamingString(o, p.strings) - if err != nil { - return o, err - } +func (s *stringValue) decode(buf []byte) ([]byte, error) { + val, o, err := msgp.ReadStringBytes(buf) + if err != nil { + return o, err + } + *s = stringValue(val) + return o, nil +} - case 7: // env - p.env, o, err = DecodeStreamingString(o, p.strings) - if err != nil { - return o, err - } +type stringTable struct { + strings []string // list of strings + indices map[string]index // map strings to their indices + nextIndex index // last index of the stringTable +} - case 8: // hostname - p.hostname, o, err = DecodeStreamingString(o, p.strings) - if err != nil { - return o, err - } +func newStringTable() *stringTable { + return &stringTable{ + strings: []string{""}, + indices: map[string]index{"": 0}, + nextIndex: 1, + } +} - case 9: // appVersion - p.appVersion, o, err = DecodeStreamingString(o, p.strings) - if err != nil { - return o, err - } - case 10: // attributes - p.attributes, o, err = DecodeKeyValueList(o, p.strings) - if err != nil { - return o, err - } - case 11: // chunks - p.chunks, o, err = DecodeTraceChunks(o, p.strings) - if err != nil { - return o, err - } - } +func (s *stringTable) Add(str string) (idx index) { + if _, ok := s.indices[str]; ok { + return s.indices[str] } - return o, nil + s.indices[str] = s.nextIndex + s.strings = append(s.strings, str) + idx = s.nextIndex + s.nextIndex += 1 + return } -func DecodeStringTable(b []byte, strings *stringTable) ([]byte, error) { - len, o, err := msgp.ReadBytesHeader(b) - if err != nil { - return nil, err +func (s *stringTable) Get(str string) (index, bool) { + if idx, ok := s.indices[str]; ok { + return idx, true } + return -1, false +} - for len > 0 { - len-- - str, o, err := msgp.ReadStringBytes(o) +func (s *stringTable) Read(b []byte) (string, []byte, bool) { + sType := getStreamingType(b[0]) + if sType == -1 { + return "", b, false + } + // if b is a string + if sType == 0 { + var sv stringValue + o, err := sv.decode(b) if err != nil { - return o, err + return "", o, false } + str := string(sv) + s.Add(str) + return str, o, true + } + // if b is an index + var i index + o, err := i.decode(b) + if err != nil { + return "", o, false + } + return s.strings[i], o, true +} - // if we've seen the string before, skip - if _, ok := strings.indices[str]; ok { - continue +// returns 0 if the given byte is a string, +// 1 if it is an int32, and -1 if it is neither. +func getStreamingType(b byte) int { + switch b { + // String formats + case 0xd9, 0xda, 0xdb: // str8, str16, str32 + return 0 + case 0xce: // uint32 + return 1 + default: + // Check for fixstr + if b&0xe0 == 0xa0 { + return 0 } - - strings.Add(str) + // Check for positive fixint + if b&0x80 == 0 { + return 1 + } + return -1 } - return o, nil } -func DecodeStreamingString(b []byte, strings *stringTable) (string, []byte, error) { - if len(b) == 0 { - return "", nil, msgp.WrapError(nil, "expected streaming string, got EOF") - } - // try reading as a uint32 index - idx, o, err := msgp.ReadUint32Bytes(b) - if err == nil { - return strings.strings[idx], o, nil - } +// traceChunk represents a list of spans with the same trace ID, +// i.e. a chunk of a trace +type traceChunk struct { + // bitmap to track which fields have been set in the trace chunk + bm bitmap - // else, try reading as a string, then add to the string table - str, o, err := msgp.ReadStringBytes(o) - if err != nil { - return "", nil, msgp.WrapError(err, "unable to read streaming string") - } - strings.Add(str) - return str, o, nil + // number of fields in the trace chunk + fields uint32 + + // the sampling priority of the trace + priority int32 + + // the optional string origin ("lambda", "rum", etc.) of the trace chunk + origin string + + // a collection of key to value pairs common in all `spans` + attributes map[string]anyValue + + // a list of spans in this chunk + spans spanList + + // whether the trace only contains analyzed spans + // (not required by tracers and set by the agent) + droppedTrace bool + + // the ID of the trace to which all spans in this chunk belong + traceID []byte + + // the optional string decision maker (previously span tag _dd.p.dm) + samplingMechanism string } +// Decoding Functions + func DecodeAnyValue(b []byte, strings *stringTable) (anyValue, []byte, error) { vType, o, err := msgp.ReadInt32Bytes(b) if err != nil { @@ -753,9 +815,9 @@ func DecodeAnyValue(b []byte, strings *stringTable) (anyValue, []byte, error) { } switch vType { case StringValueType: - str, o, err := msgp.ReadStringBytes(o) - if err != nil { - return anyValue{}, o, err + str, o, ok := strings.Read(o) + if !ok { + return anyValue{}, o, fmt.Errorf("unable to read string value") } return anyValue{valueType: StringValueType, value: str}, o, nil case BoolValueType: @@ -806,377 +868,23 @@ func DecodeAnyValue(b []byte, strings *stringTable) (anyValue, []byte, error) { } } -func DecodeKeyValueList(b []byte, strings *stringTable) (keyValueList, []byte, error) { - len, o, err := msgp.ReadMapHeaderBytes(b) +func DecodeKeyValueList(b []byte, strings *stringTable) (map[string]anyValue, []byte, error) { + numFields, o, err := msgp.ReadMapHeaderBytes(b) if err != nil { return nil, o, err } - if len == 0 || len%3 != 0 { - return nil, o, msgp.WrapError(fmt.Errorf("invalid number of items in keyValueList encoding, expected multiple of 3, got %d", len)) - } - - kv := make(keyValueList, len/3) - for i := range len / 3 { - len-- - key, o, err := DecodeStreamingString(o, strings) - if err != nil { - return nil, o, err + kv := map[string]anyValue{} + for i := range numFields { + key, o, ok := strings.Read(o) + if !ok { + return nil, o, fmt.Errorf("unable to read key of field %d", i) } - v, o, err := DecodeAnyValue(o, strings) + value, o, err := DecodeAnyValue(o, strings) if err != nil { return nil, o, err } - kv[i] = keyValue{key: streamingKey{isString: true, stringValue: key}, value: anyValue{valueType: v.valueType, value: v.value}} + kv[key] = value } return kv, o, nil } - -func DecodeTraceChunks(b []byte, strings *stringTable) ([]traceChunk, []byte, error) { - len, o, err := msgp.ReadMapHeaderBytes(b) - if err != nil { - return nil, o, err - } - - ret := make([]traceChunk, len) - for i := range len { - fields, o, err := msgp.ReadArrayHeaderBytes(o) - if err != nil { - return nil, o, err - } - tc := traceChunk{} - for fields > 0 { - fields-- - - f, o, err := msgp.ReadUint32Bytes(o) - if err != nil { - return ret, o, err - } - - switch f { - case 1: // priority - s, o, err := msgp.ReadInt32Bytes(o) - if err != nil { - return ret, o, err - } - tc.priority = s - case 2: // origin - s, o, err := msgp.ReadStringBytes(o) - if err != nil { - return ret, o, err - } - tc.origin = s - case 3: // attributes - kv, o, err := DecodeKeyValueList(o, strings) - if err != nil { - return ret, o, err - } - tc.attributes = kv - case 4: // spans - s, o, err := DecodeSpanList(o, strings) - if err != nil { - return ret, o, err - } - tc.spans = s - case 5: // droppedTrace - s, o, err := msgp.ReadBoolBytes(o) - if err != nil { - return ret, o, err - } - tc.droppedTrace = s - case 6: // traceID - s, o, err := msgp.ReadBytesBytes(o, nil) - if err != nil { - return ret, o, err - } - tc.traceID = []byte(s) - case 7: // samplingMechanism - s, o, err := msgp.ReadStringBytes(o) - if err != nil { - return ret, o, err - } - tc.samplingMechanism = s - } - } - ret[i] = tc - } - return ret, o, nil -} - -func DecodeSpanList(b []byte, strings *stringTable) (spanList, []byte, error) { - len, o, err := msgp.ReadArrayHeaderBytes(b) - if err != nil { - return nil, o, err - } - ret := make([]*Span, len) - for i := range len { - ret[i], o, err = DecodeSpan(o, strings) - if err != nil { - return nil, o, err - } - } - return ret, o, nil -} - -func DecodeSpan(b []byte, strings *stringTable) (*Span, []byte, error) { - sp := Span{} - fields, o, err := msgp.ReadMapHeaderBytes(b) - if err != nil { - return &sp, o, err - } - - for fields > 0 { - fields-- - - f, o, err := msgp.ReadUint32Bytes(o) - if err != nil { - return &sp, o, err - } - - switch f { - case 1: // service - st, o, err := msgp.ReadStringBytes(o) - if err != nil { - return &sp, o, err - } - sp.service = st - case 2: // name - st, o, err := msgp.ReadStringBytes(o) - if err != nil { - return &sp, o, err - } - sp.name = st - case 3: // resource - st, o, err := msgp.ReadStringBytes(o) - if err != nil { - return &sp, o, err - } - sp.resource = st - case 4: // spanID - i, o, err := msgp.ReadInt64Bytes(o) - if err != nil { - return &sp, o, err - } - sp.spanID = uint64(i) - case 5: // parentID - i, o, err := msgp.ReadInt64Bytes(o) - if err != nil { - return &sp, o, err - } - sp.parentID = uint64(i) - case 6: // start - i, o, err := msgp.ReadInt64Bytes(o) - if err != nil { - return &sp, o, err - } - sp.start = i - case 7: // duration - i, o, err := msgp.ReadInt64Bytes(o) - if err != nil { - return &sp, o, err - } - sp.duration = i - case 8: // error - i, o, err := msgp.ReadBoolBytes(o) - if err != nil { - return &sp, o, err - } - if i { - sp.error = 1 - } else { - sp.error = 0 - } - case 9: // attributes - kv, o, err := DecodeKeyValueList(o, strings) - if err != nil { - return &sp, o, err - } - for k, v := range kv { - key := strings.strings[k] - sp.SetTag(key, v.value.value) - } - case 10: // type - st, o, err := msgp.ReadStringBytes(o) - if err != nil { - return &sp, o, err - } - sp.spanType = st - case 11: // spanLinks - sl, o, err := DecodeSpanLinks(o, strings) - if err != nil { - return &sp, o, err - } - sp.spanLinks = sl - case 12: // spanEvents - se, o, err := DecodeSpanEvents(o, strings) - if err != nil { - return &sp, o, err - } - sp.spanEvents = se - case 13: // env - s, o, err := msgp.ReadStringBytes(o) - if err != nil { - return &sp, o, err - } - sp.SetTag(ext.Environment, s) - case 14: // version - s, o, err := msgp.ReadStringBytes(o) - if err != nil { - return &sp, o, err - } - sp.setMeta(ext.Version, s) - case 15: // component - s, o, err := msgp.ReadStringBytes(o) - if err != nil { - return &sp, o, err - } - sp.integration = s - } - } - return &sp, nil, nil -} - -func DecodeSpanLinks(b []byte, strings *stringTable) ([]SpanLink, []byte, error) { - numSpanLinks, o, err := msgp.ReadArrayHeaderBytes(b) - if err != nil { - return nil, o, err - } - - ret := make([]SpanLink, numSpanLinks) - for i := range numSpanLinks { - sl := SpanLink{} - fields, o, err := msgp.ReadMapHeaderBytes(o) - if err != nil { - return ret, o, err - } - for fields > 0 { - fields-- - - f, o, err := msgp.ReadUint32Bytes(o) - if err != nil { - return ret, o, err - } - - switch f { - case 1: // traceID - s, o, err := msgp.ReadInt64Bytes(o) - if err != nil { - return ret, o, err - } - sl.TraceID = uint64(s) - case 2: // spanID - s, o, err := msgp.ReadInt64Bytes(o) - if err != nil { - return ret, o, err - } - sl.SpanID = uint64(s) - case 3: // attributes - kv, o, err := DecodeKeyValueList(o, strings) - if err != nil { - return ret, o, err - } - for k, v := range kv { - key := strings.strings[k] - s, ok := v.value.value.(string) - if !ok { - err := msgp.WrapError(fmt.Errorf("expected string value type for span link attributes, got %T", v.value.value)) - return ret, o, err - } - sl.Attributes[key] = s - } - case 4: // tracestate - s, o, err := msgp.ReadStringBytes(o) - if err != nil { - return ret, o, err - } - sl.Tracestate = s - case 5: // flags - s, o, err := msgp.ReadUint32Bytes(o) - if err != nil { - return ret, o, err - } - sl.Flags = s - } - } - ret[i] = sl - } - return ret, o, nil -} - -func DecodeSpanEvents(b []byte, strings *stringTable) ([]spanEvent, []byte, error) { - numSpanEvents, o, err := msgp.ReadArrayHeaderBytes(b) - if err != nil { - return nil, o, err - } - ret := make([]spanEvent, numSpanEvents) - for i := range numSpanEvents { - se := spanEvent{} - fields, o, err := msgp.ReadMapHeaderBytes(o) - if err != nil { - return ret, o, err - } - for fields > 0 { - fields-- - - f, o, err := msgp.ReadUint32Bytes(o) - if err != nil { - return ret, o, err - } - - switch f { - case 1: // time - s, o, err := msgp.ReadInt64Bytes(o) - if err != nil { - return ret, o, err - } - se.TimeUnixNano = uint64(s) - case 2: // name - s, o, err := msgp.ReadStringBytes(o) - if err != nil { - return ret, o, err - } - se.Name = s - case 4: // attributes - kv, o, err := DecodeKeyValueList(o, strings) - if err != nil { - return ret, o, err - } - for k, v := range kv { - key := strings.strings[k] - switch v.value.valueType { - case StringValueType: - se.Attributes[key] = &spanEventAttribute{ - Type: spanEventAttributeTypeString, - StringValue: v.value.value.(string), - } - case BoolValueType: - se.Attributes[key] = &spanEventAttribute{ - Type: spanEventAttributeTypeBool, - BoolValue: v.value.value.(bool), - } - case IntValueType: - se.Attributes[key] = &spanEventAttribute{ - Type: spanEventAttributeTypeInt, - IntValue: v.value.value.(int64), - } - case FloatValueType: - se.Attributes[key] = &spanEventAttribute{ - Type: spanEventAttributeTypeDouble, - DoubleValue: v.value.value.(float64), - } - case ArrayValueType: - se.Attributes[key] = &spanEventAttribute{ - Type: spanEventAttributeTypeArray, - ArrayValue: v.value.value.(*spanEventArrayAttribute), - } - default: - err := msgp.WrapError(fmt.Errorf("unexpected value type not supported by span events: %T", v.value.value)) - return ret, o, err - } - } - } - } - ret[i] = se - } - return ret, nil, nil -} From 6d10c7ae892fefb9a2b74a3c64f35f740c27ddd8 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Tue, 7 Oct 2025 17:59:11 -0400 Subject: [PATCH 26/75] some fixes and more todos --- ddtrace/tracer/payload_v1.go | 46 ++++++++---------------------------- 1 file changed, 10 insertions(+), 36 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 02c5670945..8305c487a8 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -119,6 +119,7 @@ func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { attributes[k] = av p.attributes[k] = av } + // TODO(hannahkm): :sad-clown: :dead-tired: // for k, v := range span.metaStruct { // attributes = append(attributes, keyValue{key: k, value: anyValue{valueType: keyValueListType, value: v}}) // } @@ -173,7 +174,7 @@ func (p *payloadV1) clear() { func (p *payloadV1) recordItem() { atomic.AddUint32(&p.count, 1) - // p.updateHeader() TODO(hannahkm): figure out + // p.updateHeader() TODO(hannahkm): figure out if we need this } func (p *payloadV1) stats() payloadStats { @@ -270,7 +271,7 @@ func (p *payloadV1) encodeField(bm bitmap, fieldID int, a anyValue, st *stringTa return } p.buf = msgp.AppendUint32(p.buf, uint32(fieldID)) // msgp key - // p.buf = msgp.AppendInt32(p.buf, int32(a.valueType)) // value type + // p.buf = msgp.AppendInt32(p.buf, int32(a.valueType)) // value type TODO(hannahkm): do we need this? if a.valueType == StringValueType { value := a.value.(string) // encode msgp value, either by pulling from string table or writing it directly @@ -322,6 +323,7 @@ func (p *payloadV1) encodeAttributes(fieldID int, kv map[string]anyValue, st *st return nil } +// TODO(hannahkm): this references chunk.bm, which is not implemented yet func (p *payloadV1) encodeTraceChunks(fieldID int, tc []traceChunk, st *stringTable) error { if len(tc) == 0 { return nil @@ -333,23 +335,10 @@ func (p *payloadV1) encodeTraceChunks(fieldID int, tc []traceChunk, st *stringTa p.buf = msgp.AppendMapHeader(p.buf, uint32(chunk.fields)) // number of item pairs in map // priority - if chunk.bm.contains(1) { - p.buf = msgp.AppendUint32(p.buf, 1) // field ID - p.buf = msgp.AppendInt32(p.buf, chunk.priority) - } + p.encodeField(chunk.bm, 1, anyValue{valueType: IntValueType, value: chunk.priority}, st) // origin - if chunk.bm.contains(2) { - p.buf = msgp.AppendUint32(p.buf, 2) // field ID - // encode msgp value, either by pulling from string table or writing it directly - if idx, ok := st.Get(chunk.origin); ok { - p.buf = idx.encode(p.buf) - } else { - s := stringValue(chunk.origin) - p.buf = s.encode(p.buf) - st.Add(chunk.origin) - } - } + p.encodeField(chunk.bm, 2, anyValue{valueType: StringValueType, value: chunk.origin}, st) // attributes if chunk.bm.contains(3) { @@ -362,29 +351,14 @@ func (p *payloadV1) encodeTraceChunks(fieldID int, tc []traceChunk, st *stringTa } // droppedTrace - if chunk.bm.contains(5) { - p.buf = msgp.AppendUint32(p.buf, 5) // field ID - p.buf = msgp.AppendBool(p.buf, chunk.droppedTrace) - } + p.encodeField(chunk.bm, 5, anyValue{valueType: BoolValueType, value: chunk.droppedTrace}, st) // traceID - if chunk.bm.contains(6) { - p.buf = msgp.AppendUint32(p.buf, 6) // field ID - p.buf = msgp.AppendBytes(p.buf, chunk.traceID) - } + p.encodeField(chunk.bm, 6, anyValue{valueType: BytesValueType, value: chunk.traceID}, st) // samplingMechanism - if chunk.bm.contains(7) { - p.buf = msgp.AppendUint32(p.buf, 7) // field ID - // encode msgp value, either by pulling from string table or writing it directly - if idx, ok := st.Get(chunk.samplingMechanism); ok { - p.buf = idx.encode(p.buf) - } else { - s := stringValue(chunk.samplingMechanism) - p.buf = s.encode(p.buf) - st.Add(chunk.samplingMechanism) - } - } + // TODO(hannahkm): I think the RFC changed, need to double check this + p.encodeField(chunk.bm, 7, anyValue{valueType: StringValueType, value: chunk.samplingMechanism}, st) } return nil From 46a6cb92253b5cc21c1a1b192097a12296fbfd25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dario=20Casta=C3=B1=C3=A9?= Date: Wed, 8 Oct 2025 17:41:00 +0200 Subject: [PATCH 27/75] fix(ddtrace/tracer): improving the ergonomics of encodeField and small editions --- ddtrace/tracer/payload_v1.go | 116 ++++++++++++++++++----------------- 1 file changed, 61 insertions(+), 55 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 8305c487a8..df41543ab5 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -110,6 +110,7 @@ func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { continue } for k, v := range span.meta { + // TODO(darccio): aren't these loops potentially overriding tags from different spans as attributes? av := anyValue{valueType: StringValueType, value: v} attributes[k] = av p.attributes[k] = av @@ -175,6 +176,7 @@ func (p *payloadV1) clear() { func (p *payloadV1) recordItem() { atomic.AddUint32(&p.count, 1) // p.updateHeader() TODO(hannahkm): figure out if we need this + // TODO(darccio): after updating updateHeader, we should do it when we bump p.fields. } func (p *payloadV1) stats() payloadStats { @@ -197,7 +199,7 @@ func (p *payloadV1) protocol() float64 { } func (p *payloadV1) updateHeader() { - n := uint64(p.fields) + n := uint64(p.fields) // TODO(darccio): possibly this needs to be atomic as it was before with p.count? switch { case n <= 15: p.header[7] = msgpackMapFix + byte(n) @@ -247,14 +249,14 @@ func (p *payloadV1) Read(b []byte) (n int, err error) { // encode writes existing payload fields into the buffer in msgp format. func (p *payloadV1) encode() { st := newStringTable() - p.encodeField(p.bm, 2, anyValue{valueType: StringValueType, value: p.containerID}, st) - p.encodeField(p.bm, 3, anyValue{valueType: StringValueType, value: p.languageName}, st) - p.encodeField(p.bm, 4, anyValue{valueType: StringValueType, value: p.languageVersion}, st) - p.encodeField(p.bm, 5, anyValue{valueType: StringValueType, value: p.tracerVersion}, st) - p.encodeField(p.bm, 6, anyValue{valueType: StringValueType, value: p.runtimeID}, st) - p.encodeField(p.bm, 7, anyValue{valueType: StringValueType, value: p.env}, st) - p.encodeField(p.bm, 8, anyValue{valueType: StringValueType, value: p.hostname}, st) - p.encodeField(p.bm, 9, anyValue{valueType: StringValueType, value: p.appVersion}, st) + p.buf = encodeField(p.buf, p.bm, 2, p.containerID, st) + p.buf = encodeField(p.buf, p.bm, 3, p.languageName, st) + p.buf = encodeField(p.buf, p.bm, 4, p.languageVersion, st) + p.buf = encodeField(p.buf, p.bm, 5, p.tracerVersion, st) + p.buf = encodeField(p.buf, p.bm, 6, p.runtimeID, st) + p.buf = encodeField(p.buf, p.bm, 7, p.env, st) + p.buf = encodeField(p.buf, p.bm, 8, p.hostname, st) + p.buf = encodeField(p.buf, p.bm, 9, p.appVersion, st) if len(p.attributes) > 0 { p.encodeAttributes(10, p.attributes, st) @@ -265,40 +267,41 @@ func (p *payloadV1) encode() { } } +type fieldValue interface { + bool | []byte | int32 | int64 | uint32 | uint64 | string +} + // TODO(hannahkm): is this the best way to go about encoding fields? -func (p *payloadV1) encodeField(bm bitmap, fieldID int, a anyValue, st *stringTable) { - if !bm.contains(uint32(fieldID)) { - return +func encodeField[F fieldValue](buf []byte, bm bitmap, fieldID uint32, a F, st *stringTable) []byte { + if !bm.contains(fieldID) { + return buf } - p.buf = msgp.AppendUint32(p.buf, uint32(fieldID)) // msgp key - // p.buf = msgp.AppendInt32(p.buf, int32(a.valueType)) // value type TODO(hannahkm): do we need this? - if a.valueType == StringValueType { - value := a.value.(string) + buf = msgp.AppendUint32(buf, uint32(fieldID)) // msgp key + switch value := any(a).(type) { + case string: // encode msgp value, either by pulling from string table or writing it directly if idx, ok := st.Get(value); ok { - p.buf = idx.encode(p.buf) + buf = idx.encode(buf) } else { s := stringValue(value) - p.buf = s.encode(p.buf) + buf = s.encode(buf) st.Add(value) } - return - } - switch a.valueType { - case BoolValueType: - p.buf = msgp.AppendBool(p.buf, a.value.(bool)) - case FloatValueType: - p.buf = msgp.AppendFloat64(p.buf, a.value.(float64)) - case IntValueType: - p.buf = msgp.AppendInt64(p.buf, a.value.(int64)) - case BytesValueType: - p.buf = msgp.AppendBytes(p.buf, a.value.([]byte)) - case ArrayValueType: - p.buf = msgp.AppendArrayHeader(p.buf, uint32(len(a.value.(arrayValue)))) - for _, v := range a.value.(arrayValue) { - v.encode(p.buf) + case bool: + buf = msgp.AppendBool(buf, value) + case float64: + buf = msgp.AppendFloat64(buf, value) + case int32, int64: + buf = msgp.AppendInt64(buf, value.(int64)) + case []byte: + buf = msgp.AppendBytes(buf, value) + case arrayValue: + buf = msgp.AppendArrayHeader(buf, uint32(len(value))) + for _, v := range value { + buf = v.encode(buf) } } + return buf } func (p *payloadV1) encodeAttributes(fieldID int, kv map[string]anyValue, st *stringTable) error { @@ -324,6 +327,7 @@ func (p *payloadV1) encodeAttributes(fieldID int, kv map[string]anyValue, st *st } // TODO(hannahkm): this references chunk.bm, which is not implemented yet +// TODO(darccio): we can go without chunk.bm or other bm down the line func (p *payloadV1) encodeTraceChunks(fieldID int, tc []traceChunk, st *stringTable) error { if len(tc) == 0 { return nil @@ -335,10 +339,10 @@ func (p *payloadV1) encodeTraceChunks(fieldID int, tc []traceChunk, st *stringTa p.buf = msgp.AppendMapHeader(p.buf, uint32(chunk.fields)) // number of item pairs in map // priority - p.encodeField(chunk.bm, 1, anyValue{valueType: IntValueType, value: chunk.priority}, st) + p.buf = encodeField(p.buf, chunk.bm, 1, chunk.priority, st) // origin - p.encodeField(chunk.bm, 2, anyValue{valueType: StringValueType, value: chunk.origin}, st) + p.buf = encodeField(p.buf, chunk.bm, 2, chunk.origin, st) // attributes if chunk.bm.contains(3) { @@ -351,14 +355,14 @@ func (p *payloadV1) encodeTraceChunks(fieldID int, tc []traceChunk, st *stringTa } // droppedTrace - p.encodeField(chunk.bm, 5, anyValue{valueType: BoolValueType, value: chunk.droppedTrace}, st) + p.buf = encodeField(p.buf, chunk.bm, 5, chunk.droppedTrace, st) // traceID - p.encodeField(chunk.bm, 6, anyValue{valueType: BytesValueType, value: chunk.traceID}, st) + p.buf = encodeField(p.buf, chunk.bm, 6, chunk.traceID, st) // samplingMechanism // TODO(hannahkm): I think the RFC changed, need to double check this - p.encodeField(chunk.bm, 7, anyValue{valueType: StringValueType, value: chunk.samplingMechanism}, st) + p.buf = encodeField(p.buf, chunk.bm, 7, chunk.samplingMechanism, st) } return nil @@ -378,22 +382,22 @@ func (p *payloadV1) encodeSpans(fieldID int, spans spanList, st *stringTable) er } // TODO(hannahkm): how do we get the number of set fields efficiently? // TODO(hannahkm): might need to change the bitmap value in the calls below - p.encodeField(p.bm, 1, anyValue{valueType: StringValueType, value: span.service}, st) - p.encodeField(p.bm, 2, anyValue{valueType: StringValueType, value: span.name}, st) - p.encodeField(p.bm, 3, anyValue{valueType: StringValueType, value: span.resource}, st) - p.encodeField(p.bm, 4, anyValue{valueType: IntValueType, value: span.spanID}, st) - p.encodeField(p.bm, 5, anyValue{valueType: IntValueType, value: span.parentID}, st) - p.encodeField(p.bm, 6, anyValue{valueType: IntValueType, value: span.start}, st) - p.encodeField(p.bm, 7, anyValue{valueType: IntValueType, value: span.duration}, st) + p.buf = encodeField(p.buf, p.bm, 1, span.service, st) + p.buf = encodeField(p.buf, p.bm, 2, span.name, st) + p.buf = encodeField(p.buf, p.bm, 3, span.resource, st) + p.buf = encodeField(p.buf, p.bm, 4, span.spanID, st) + p.buf = encodeField(p.buf, p.bm, 5, span.parentID, st) + p.buf = encodeField(p.buf, p.bm, 6, span.start, st) + p.buf = encodeField(p.buf, p.bm, 7, span.duration, st) if span.error != 0 { - p.encodeField(p.bm, 8, anyValue{valueType: BoolValueType, value: true}, st) + p.buf = encodeField(p.buf, p.bm, 8, true, st) } else { - p.encodeField(p.bm, 8, anyValue{valueType: BoolValueType, value: false}, st) + p.buf = encodeField(p.buf, p.bm, 8, false, st) } - p.encodeField(p.bm, 10, anyValue{valueType: StringValueType, value: span.spanType}, st) + p.buf = encodeField(p.buf, p.bm, 10, span.spanType, st) p.encodeSpanLinks(11, span.spanLinks, st) p.encodeSpanEvents(12, span.spanEvents, st) - p.encodeField(p.bm, 15, anyValue{valueType: StringValueType, value: span.integration}, st) + p.buf = encodeField(p.buf, p.bm, 15, span.integration, st) // TODO(hannahkm): add attributes, env, version } @@ -410,10 +414,10 @@ func (p *payloadV1) encodeSpanLinks(fieldID int, spanLinks []SpanLink, st *strin for _, link := range spanLinks { // TODO(hannahkm): how do we get the number of set fields // TODO(hannahkm): might need to change the bitmap value in the calls below - p.encodeField(p.bm, 1, anyValue{valueType: BytesValueType, value: link.TraceID}, st) - p.encodeField(p.bm, 2, anyValue{valueType: IntValueType, value: link.SpanID}, st) - p.encodeField(p.bm, 4, anyValue{valueType: StringValueType, value: link.Tracestate}, st) - p.encodeField(p.bm, 5, anyValue{valueType: StringValueType, value: link.Flags}, st) + p.buf = encodeField(p.buf, p.bm, 1, link.TraceID, st) + p.buf = encodeField(p.buf, p.bm, 2, link.SpanID, st) + p.buf = encodeField(p.buf, p.bm, 4, link.Tracestate, st) + p.buf = encodeField(p.buf, p.bm, 5, link.Flags, st) // TODO(hannahkm): add attributes } @@ -430,8 +434,8 @@ func (p *payloadV1) encodeSpanEvents(fieldID int, spanEvents []spanEvent, st *st for _, event := range spanEvents { // TODO(hannahkm): how do we get the number of set fields // TODO(hannahkm): might need to change the bitmap value in the calls below - p.encodeField(p.bm, 1, anyValue{valueType: IntValueType, value: event.TimeUnixNano}, st) - p.encodeField(p.bm, 2, anyValue{valueType: StringValueType, value: event.Name}, st) + p.buf = encodeField(p.buf, p.bm, 1, event.TimeUnixNano, st) + p.buf = encodeField(p.buf, p.bm, 2, event.Name, st) // TODO(hannahkm): add attributes } return nil @@ -777,6 +781,8 @@ type traceChunk struct { traceID []byte // the optional string decision maker (previously span tag _dd.p.dm) + // TODO(darccio): we need to make sure this is an uint32 and the value assigned to _dd.p.m + // is set here but always positive. samplingMechanism string } From 170a42c026ce077fee1bb8fe73e16d4a0849f380 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Wed, 8 Oct 2025 16:17:19 -0400 Subject: [PATCH 28/75] wip: finish encoding --- ddtrace/tracer/payload_v1.go | 223 ++++++++++++++++++++++++----------- 1 file changed, 154 insertions(+), 69 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index df41543ab5..c7f6c5199b 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -9,8 +9,11 @@ import ( "bytes" "encoding/binary" "fmt" + "strconv" "sync/atomic" + "github.com/DataDog/dd-trace-go/v2/ddtrace/ext" + "github.com/DataDog/dd-trace-go/v2/internal/log" "github.com/tinylib/msgp/msgp" ) @@ -104,31 +107,34 @@ func newPayloadV1() *payloadV1 { func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { // We need to hydrate the payload with everything we get from the spans. // Conceptually, our `t spanList` corresponds to one `traceChunk`. + + // For now, we blindly set the origin, priority, and attributes values for the chunk + // In the future, attributes should hold values that are shared across all chunks in the payload attributes := map[string]anyValue{} + origin, priority, sm := "", 0, 0 for _, span := range t { if span == nil { - continue - } - for k, v := range span.meta { - // TODO(darccio): aren't these loops potentially overriding tags from different spans as attributes? - av := anyValue{valueType: StringValueType, value: v} - attributes[k] = av - p.attributes[k] = av + break } - for k, v := range span.metrics { - av := anyValue{valueType: FloatValueType, value: v} - attributes[k] = av - p.attributes[k] = av + if p, ok := span.Context().SamplingPriority(); ok { + origin = span.Context().origin + priority = p + attributes["service"] = anyValue{valueType: StringValueType, value: span.Root().service} + dm := span.context.trace.propagatingTag(keyDecisionMaker) + sm, err = strconv.Atoi(dm) + if err != nil { + log.Error("failed to convert decision maker to int: %s", err.Error()) + } + break } - // TODO(hannahkm): :sad-clown: :dead-tired: - // for k, v := range span.metaStruct { - // attributes = append(attributes, keyValue{key: k, value: anyValue{valueType: keyValueListType, value: v}}) - // } } tc := traceChunk{ - spans: t, - traceID: t[0].Context().traceID[:], + spans: t, + priority: int32(priority), + origin: origin, + traceID: t[0].Context().traceID[:], + samplingMechanism: uint32(sm), } // if there are attributes available, set them in our bitmap and increment @@ -175,8 +181,6 @@ func (p *payloadV1) clear() { func (p *payloadV1) recordItem() { atomic.AddUint32(&p.count, 1) - // p.updateHeader() TODO(hannahkm): figure out if we need this - // TODO(darccio): after updating updateHeader, we should do it when we bump p.fields. } func (p *payloadV1) stats() payloadStats { @@ -199,17 +203,17 @@ func (p *payloadV1) protocol() float64 { } func (p *payloadV1) updateHeader() { - n := uint64(p.fields) // TODO(darccio): possibly this needs to be atomic as it was before with p.count? + n := atomic.LoadUint32(&p.fields) switch { case n <= 15: p.header[7] = msgpackMapFix + byte(n) p.readOff = 7 case n <= 1<<16-1: - binary.BigEndian.PutUint64(p.header, n) // writes 2 bytes + binary.BigEndian.PutUint64(p.header, uint64(n)) // writes 2 bytes p.header[5] = msgpackMap16 p.readOff = 5 default: // n <= 1<<32-1 - binary.BigEndian.PutUint64(p.header, n) // writes 4 bytes + binary.BigEndian.PutUint64(p.header, uint64(n)) // writes 4 bytes p.header[3] = msgpackMap32 p.readOff = 3 } @@ -271,7 +275,8 @@ type fieldValue interface { bool | []byte | int32 | int64 | uint32 | uint64 | string } -// TODO(hannahkm): is this the best way to go about encoding fields? +// encodeField takes a field of any value and encodes it into the given buffer +// in msgp format. func encodeField[F fieldValue](buf []byte, bm bitmap, fieldID uint32, a F, st *stringTable) []byte { if !bm.contains(fieldID) { return buf @@ -326,43 +331,36 @@ func (p *payloadV1) encodeAttributes(fieldID int, kv map[string]anyValue, st *st return nil } -// TODO(hannahkm): this references chunk.bm, which is not implemented yet -// TODO(darccio): we can go without chunk.bm or other bm down the line func (p *payloadV1) encodeTraceChunks(fieldID int, tc []traceChunk, st *stringTable) error { - if len(tc) == 0 { + if len(tc) == 0 || !p.bm.contains(uint32(fieldID)) { return nil } p.buf = msgp.AppendUint32(p.buf, uint32(fieldID)) // msgp key p.buf = msgp.AppendArrayHeader(p.buf, uint32(len(tc))) // number of chunks for _, chunk := range tc { - p.buf = msgp.AppendMapHeader(p.buf, uint32(chunk.fields)) // number of item pairs in map + p.buf = msgp.AppendMapHeader(p.buf, 7) // number of fields in chunk // priority - p.buf = encodeField(p.buf, chunk.bm, 1, chunk.priority, st) + p.buf = encodeField(p.buf, fullSetBitmap, 1, chunk.priority, st) // origin - p.buf = encodeField(p.buf, chunk.bm, 2, chunk.origin, st) + p.buf = encodeField(p.buf, fullSetBitmap, 2, chunk.origin, st) // attributes - if chunk.bm.contains(3) { - p.encodeAttributes(3, chunk.attributes, st) - } + p.encodeAttributes(3, chunk.attributes, st) // spans - if chunk.bm.contains(4) { - p.encodeSpans(4, chunk.spans, st) - } + p.encodeSpans(4, chunk.spans, st) // droppedTrace - p.buf = encodeField(p.buf, chunk.bm, 5, chunk.droppedTrace, st) + p.buf = encodeField(p.buf, fullSetBitmap, 5, chunk.droppedTrace, st) // traceID - p.buf = encodeField(p.buf, chunk.bm, 6, chunk.traceID, st) + p.buf = encodeField(p.buf, fullSetBitmap, 6, chunk.traceID, st) // samplingMechanism - // TODO(hannahkm): I think the RFC changed, need to double check this - p.buf = encodeField(p.buf, chunk.bm, 7, chunk.samplingMechanism, st) + p.buf = encodeField(p.buf, fullSetBitmap, 7, chunk.samplingMechanism, st) } return nil @@ -380,26 +378,54 @@ func (p *payloadV1) encodeSpans(fieldID int, spans spanList, st *stringTable) er if span == nil { continue } - // TODO(hannahkm): how do we get the number of set fields efficiently? - // TODO(hannahkm): might need to change the bitmap value in the calls below - p.buf = encodeField(p.buf, p.bm, 1, span.service, st) - p.buf = encodeField(p.buf, p.bm, 2, span.name, st) - p.buf = encodeField(p.buf, p.bm, 3, span.resource, st) - p.buf = encodeField(p.buf, p.bm, 4, span.spanID, st) - p.buf = encodeField(p.buf, p.bm, 5, span.parentID, st) - p.buf = encodeField(p.buf, p.bm, 6, span.start, st) - p.buf = encodeField(p.buf, p.bm, 7, span.duration, st) + p.buf = msgp.AppendMapHeader(p.buf, 16) // number of fields in span + + p.buf = encodeField(p.buf, fullSetBitmap, 1, span.service, st) + p.buf = encodeField(p.buf, fullSetBitmap, 2, span.name, st) + p.buf = encodeField(p.buf, fullSetBitmap, 3, span.resource, st) + p.buf = encodeField(p.buf, fullSetBitmap, 4, span.spanID, st) + p.buf = encodeField(p.buf, fullSetBitmap, 5, span.parentID, st) + p.buf = encodeField(p.buf, fullSetBitmap, 6, span.start, st) + p.buf = encodeField(p.buf, fullSetBitmap, 7, span.duration, st) if span.error != 0 { - p.buf = encodeField(p.buf, p.bm, 8, true, st) + p.buf = encodeField(p.buf, fullSetBitmap, 8, true, st) } else { - p.buf = encodeField(p.buf, p.bm, 8, false, st) + p.buf = encodeField(p.buf, fullSetBitmap, 8, false, st) + } + + // span attributes combine the meta (tags), metrics and meta_struct + attr := map[string]anyValue{} + for k, v := range span.meta { + attr[k] = anyValue{ + valueType: StringValueType, + value: stringValue(v), + } + } + for k, v := range span.metrics { + attr[k] = anyValue{ + valueType: FloatValueType, + value: v, + } + } + for k, v := range span.metaStruct { + av := buildAnyValue(v) + if av != nil { + attr[k] = *av + } } - p.buf = encodeField(p.buf, p.bm, 10, span.spanType, st) + p.encodeAttributes(9, attr, st) + + p.buf = encodeField(p.buf, fullSetBitmap, 10, span.spanType, st) p.encodeSpanLinks(11, span.spanLinks, st) p.encodeSpanEvents(12, span.spanEvents, st) - p.buf = encodeField(p.buf, p.bm, 15, span.integration, st) - // TODO(hannahkm): add attributes, env, version + env := span.meta[ext.Environment] + p.buf = encodeField(p.buf, fullSetBitmap, 13, env, st) + + version := span.meta[ext.Version] + p.buf = encodeField(p.buf, fullSetBitmap, 14, version, st) + + p.buf = encodeField(p.buf, fullSetBitmap, 15, span.integration, st) } return nil } @@ -412,14 +438,21 @@ func (p *payloadV1) encodeSpanLinks(fieldID int, spanLinks []SpanLink, st *strin p.buf = msgp.AppendArrayHeader(p.buf, uint32(len(spanLinks))) // number of span links for _, link := range spanLinks { - // TODO(hannahkm): how do we get the number of set fields - // TODO(hannahkm): might need to change the bitmap value in the calls below - p.buf = encodeField(p.buf, p.bm, 1, link.TraceID, st) - p.buf = encodeField(p.buf, p.bm, 2, link.SpanID, st) - p.buf = encodeField(p.buf, p.bm, 4, link.Tracestate, st) - p.buf = encodeField(p.buf, p.bm, 5, link.Flags, st) - - // TODO(hannahkm): add attributes + p.buf = msgp.AppendMapHeader(p.buf, 5) // number of fields in span link + + p.buf = encodeField(p.buf, fullSetBitmap, 1, link.TraceID, st) + p.buf = encodeField(p.buf, fullSetBitmap, 2, link.SpanID, st) + p.buf = encodeField(p.buf, fullSetBitmap, 4, link.Tracestate, st) + p.buf = encodeField(p.buf, fullSetBitmap, 5, link.Flags, st) + + attr := map[string]anyValue{} + for k, v := range link.Attributes { + attr[k] = anyValue{ + valueType: StringValueType, + value: stringValue(v), + } + } + p.encodeAttributes(3, attr, st) } return nil } @@ -432,11 +465,44 @@ func (p *payloadV1) encodeSpanEvents(fieldID int, spanEvents []spanEvent, st *st p.buf = msgp.AppendArrayHeader(p.buf, uint32(len(spanEvents))) // number of span events for _, event := range spanEvents { - // TODO(hannahkm): how do we get the number of set fields - // TODO(hannahkm): might need to change the bitmap value in the calls below - p.buf = encodeField(p.buf, p.bm, 1, event.TimeUnixNano, st) - p.buf = encodeField(p.buf, p.bm, 2, event.Name, st) - // TODO(hannahkm): add attributes + p.buf = msgp.AppendMapHeader(p.buf, 3) // number of fields in span event + + p.buf = encodeField(p.buf, fullSetBitmap, 1, event.TimeUnixNano, st) + p.buf = encodeField(p.buf, fullSetBitmap, 2, event.Name, st) + + attr := map[string]anyValue{} + for k, v := range event.Attributes { + switch v.Type { + case spanEventAttributeTypeString: + attr[k] = anyValue{ + valueType: StringValueType, + value: v.StringValue, + } + case spanEventAttributeTypeInt: + attr[k] = anyValue{ + valueType: IntValueType, + value: v.IntValue, + } + case spanEventAttributeTypeDouble: + attr[k] = anyValue{ + valueType: FloatValueType, + value: v.DoubleValue, + } + case spanEventAttributeTypeBool: + attr[k] = anyValue{ + valueType: BoolValueType, + value: v.BoolValue, + } + case spanEventAttributeTypeArray: + attr[k] = anyValue{ + valueType: ArrayValueType, + value: v.ArrayValue, + } + default: + log.Warn("dropped unsupported span event attribute type %d", v.Type) + } + } + p.encodeAttributes(3, attr, st) } return nil } @@ -599,6 +665,25 @@ const ( keyValueListType // []keyValue -- 7 ) +func buildAnyValue(v any) *anyValue { + switch v := v.(type) { + case string: + return &anyValue{valueType: StringValueType, value: v} + case bool: + return &anyValue{valueType: BoolValueType, value: v} + case float64: + return &anyValue{valueType: FloatValueType, value: v} + case int64, int32: + return &anyValue{valueType: IntValueType, value: v} + case []byte: + return &anyValue{valueType: BytesValueType, value: v} + case arrayValue: + return &anyValue{valueType: ArrayValueType, value: v} + default: + return nil + } +} + func (a anyValue) encode(buf []byte) []byte { buf = msgp.AppendInt32(buf, int32(a.valueType)) switch a.valueType { @@ -627,6 +712,8 @@ type arrayValue []anyValue // 1 for represented fields and 0 for unset fields. type bitmap int16 +var fullSetBitmap bitmap = -1 + func (b *bitmap) set(bit uint32) { if bit >= 16 { return @@ -781,9 +868,7 @@ type traceChunk struct { traceID []byte // the optional string decision maker (previously span tag _dd.p.dm) - // TODO(darccio): we need to make sure this is an uint32 and the value assigned to _dd.p.m - // is set here but always positive. - samplingMechanism string + samplingMechanism uint32 } // Decoding Functions From 7d6c3cc410883261c7fe2ff79fc36ca9e63b5c47 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Wed, 8 Oct 2025 17:23:15 -0400 Subject: [PATCH 29/75] decode?? --- ddtrace/tracer/payload_v1.go | 336 +++++++++++++++++++++++++++++++++-- 1 file changed, 322 insertions(+), 14 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index c7f6c5199b..9ac7f529bf 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -598,10 +598,10 @@ func (p *payloadV1) decodeBuffer() ([]byte, error) { // handle trace chunks if idx == 11 { - // p.chunks, o, err = DecodeTraceChunks(o, st) - // if err != nil { - // break - // } + p.chunks, o, err = DecodeTraceChunks(o, st) + if err != nil { + break + } continue } @@ -842,12 +842,6 @@ func getStreamingType(b byte) int { // traceChunk represents a list of spans with the same trace ID, // i.e. a chunk of a trace type traceChunk struct { - // bitmap to track which fields have been set in the trace chunk - bm bitmap - - // number of fields in the trace chunk - fields uint32 - // the sampling priority of the trace priority int32 @@ -872,8 +866,322 @@ type traceChunk struct { } // Decoding Functions +func DecodeTraceChunks(b []byte, st *stringTable) ([]traceChunk, []byte, error) { + out := []traceChunk{} + numChunks, o, err := msgp.ReadArrayHeaderBytes(b) + if err != nil { + return nil, o, err + } + for range numChunks { + tc := traceChunk{} + b, err = tc.decode(b, st) + if err != nil { + return nil, o, err + } + out = append(out, tc) + } + return out, o, nil +} + +func (tc *traceChunk) decode(b []byte, st *stringTable) ([]byte, error) { + numFields, o, err := msgp.ReadMapHeaderBytes(b) + if err != nil { + return o, err + } + for range numFields { + // read msgp field ID + var idx uint32 + idx, o, err = msgp.ReadUint32Bytes(o) + if err != nil { + return o, err + } + + // read msgp string value + switch idx { + case 1: + tc.priority, o, err = msgp.ReadInt32Bytes(o) + if err != nil { + return o, err + } + case 2: + tc.origin, o, err = msgp.ReadStringBytes(o) + if err != nil { + return o, err + } + case 3: + tc.attributes, o, err = DecodeKeyValueList(o, st) + if err != nil { + return o, err + } + case 4: + tc.spans, o, err = DecodeSpans(o, st) + if err != nil { + return o, err + } + case 5: + tc.droppedTrace, o, err = msgp.ReadBoolBytes(o) + if err != nil { + return o, err + } + case 6: + tc.traceID, o, err = msgp.ReadBytesBytes(o, nil) + if err != nil { + return o, err + } + case 7: + tc.samplingMechanism, o, err = msgp.ReadUint32Bytes(o) + if err != nil { + return o, err + } + default: + return o, fmt.Errorf("unexpected field ID %d", idx) + } + } + return o, err +} + +func DecodeSpans(b []byte, st *stringTable) (spanList, []byte, error) { + out := spanList{} + numSpans, o, err := msgp.ReadArrayHeaderBytes(b) + if err != nil { + return nil, o, err + } + for range numSpans { + span := Span{} + b, err = span.decode(b, st) + if err != nil { + return nil, o, err + } + out = append(out, &span) + } + return out, o, nil +} + +func (span *Span) decode(b []byte, st *stringTable) ([]byte, error) { + numFields, o, err := msgp.ReadMapHeaderBytes(b) + if err != nil { + return o, err + } + for range numFields { + // read msgp field ID + var idx uint32 + idx, o, err = msgp.ReadUint32Bytes(o) + if err != nil { + return o, err + } + + // read msgp string value + switch idx { + case 1: + span.name, o, err = msgp.ReadStringBytes(o) + if err != nil { + return o, err + } + case 2: + span.service, o, err = msgp.ReadStringBytes(o) + if err != nil { + return o, err + } + case 3: + span.resource, o, err = msgp.ReadStringBytes(o) + if err != nil { + return o, err + } + case 4: + span.spanType, o, err = msgp.ReadStringBytes(o) + if err != nil { + return o, err + } + case 5: + span.start, o, err = msgp.ReadInt64Bytes(o) + if err != nil { + return o, err + } + case 6: + span.duration, o, err = msgp.ReadInt64Bytes(o) + if err != nil { + return o, err + } + // case 7: + // span.meta, o, err = DecodeKeyValueList(o, st) + // if err != nil { + // return o, err + // } + // case 8: + // span.metaStruct, o, err = DecodeKeyValueList(o, st) + // if err != nil { + // return o, err + // } + // case 9: + // span.metrics, o, err = DecodeKeyValueList(o, st) + // if err != nil { + // return o, err + // } + case 10: + span.spanID, o, err = msgp.ReadUint64Bytes(o) + if err != nil { + return o, err + } + case 11: + span.traceID, o, err = msgp.ReadUint64Bytes(o) + if err != nil { + return o, err + } + case 12: + span.parentID, o, err = msgp.ReadUint64Bytes(o) + if err != nil { + return o, err + } + case 13: + var v bool + v, o, err = msgp.ReadBoolBytes(o) + if err != nil { + return o, err + } + if v { + span.error = 1 + } else { + span.error = 0 + } + case 14: + span.spanLinks, o, err = DecodeSpanLinks(o, st) + if err != nil { + return o, err + } + case 15: + span.spanEvents, o, err = DecodeSpanEvents(o, st) + if err != nil { + return o, err + } + case 16: + span.integration, o, err = msgp.ReadStringBytes(o) + if err != nil { + return o, err + } + default: + return o, fmt.Errorf("unexpected field ID %d", idx) + } + } + return o, err +} + +func DecodeSpanLinks(b []byte, st *stringTable) ([]SpanLink, []byte, error) { + out := []SpanLink{} + numLinks, o, err := msgp.ReadArrayHeaderBytes(b) + if err != nil { + return nil, o, err + } + for range numLinks { + link := SpanLink{} + b, err = link.decode(b, st) + if err != nil { + return nil, o, err + } + out = append(out, link) + } + return out, o, nil +} + +func (link *SpanLink) decode(b []byte, st *stringTable) ([]byte, error) { + numFields, o, err := msgp.ReadMapHeaderBytes(b) + if err != nil { + return o, err + } + for range numFields { + // read msgp field ID + var idx uint32 + idx, o, err = msgp.ReadUint32Bytes(o) + if err != nil { + return o, err + } + + // read msgp string value + switch idx { + case 1: + link.TraceID, o, err = msgp.ReadUint64Bytes(o) + if err != nil { + return o, err + } + case 2: + link.SpanID, o, err = msgp.ReadUint64Bytes(o) + if err != nil { + return o, err + } + // case 3: + // link.Attributes, o, err = DecodeKeyValueList(o, st) + // if err != nil { + // return o, err + // } + case 4: + link.Tracestate, o, err = msgp.ReadStringBytes(o) + if err != nil { + return o, err + } + case 5: + link.Flags, o, err = msgp.ReadUint32Bytes(o) + if err != nil { + return o, err + } + default: + return o, fmt.Errorf("unexpected field ID %d", idx) + } + } + return o, err +} + +func DecodeSpanEvents(b []byte, st *stringTable) ([]spanEvent, []byte, error) { + out := []spanEvent{} + numEvents, o, err := msgp.ReadArrayHeaderBytes(b) + if err != nil { + return nil, o, err + } + for range numEvents { + event := spanEvent{} + b, err = event.decode(b, st) + if err != nil { + return nil, o, err + } + out = append(out, event) + } + return out, o, nil +} + +func (event *spanEvent) decode(b []byte, st *stringTable) ([]byte, error) { + numFields, o, err := msgp.ReadMapHeaderBytes(b) + if err != nil { + return o, err + } + for range numFields { + // read msgp field ID + var idx uint32 + idx, o, err = msgp.ReadUint32Bytes(o) + if err != nil { + return o, err + } + switch idx { + case 1: + event.TimeUnixNano, o, err = msgp.ReadUint64Bytes(o) + if err != nil { + return o, err + } + case 2: + event.Name, o, err = msgp.ReadStringBytes(o) + if err != nil { + return o, err + } + // case 3: + // event.Attributes, o, err = DecodeKeyValueList(o, st) + // if err != nil { + // break + // } + default: + return o, fmt.Errorf("unexpected field ID %d", idx) + } + } + return o, err +} -func DecodeAnyValue(b []byte, strings *stringTable) (anyValue, []byte, error) { +func decodeAnyValue(b []byte, strings *stringTable) (anyValue, []byte, error) { vType, o, err := msgp.ReadInt32Bytes(b) if err != nil { return anyValue{}, o, err @@ -916,7 +1224,7 @@ func DecodeAnyValue(b []byte, strings *stringTable) (anyValue, []byte, error) { } arrayValue := make(arrayValue, len/2) for i := range len / 2 { - arrayValue[i], o, err = DecodeAnyValue(o, strings) + arrayValue[i], o, err = decodeAnyValue(o, strings) if err != nil { return anyValue{}, o, err } @@ -945,11 +1253,11 @@ func DecodeKeyValueList(b []byte, strings *stringTable) (map[string]anyValue, [] if !ok { return nil, o, fmt.Errorf("unable to read key of field %d", i) } - value, o, err := DecodeAnyValue(o, strings) + av, o, err := decodeAnyValue(o, strings) if err != nil { return nil, o, err } - kv[key] = value + kv[key] = av } return kv, o, nil } From 5881a508a71daa0d867b2fae1dd47fd417e8b8f8 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Wed, 8 Oct 2025 19:01:56 -0400 Subject: [PATCH 30/75] debug some issues --- ddtrace/tracer/payload_v1.go | 99 +++++++++++++++++++++--------------- 1 file changed, 58 insertions(+), 41 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 9ac7f529bf..05f0a4d94a 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -107,6 +107,10 @@ func newPayloadV1() *payloadV1 { func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { // We need to hydrate the payload with everything we get from the spans. // Conceptually, our `t spanList` corresponds to one `traceChunk`. + if !p.bm.contains(11) && len(t) > 0 { + p.bm.set(11) + p.fields += 1 + } // For now, we blindly set the origin, priority, and attributes values for the chunk // In the future, attributes should hold values that are shared across all chunks in the payload @@ -139,11 +143,12 @@ func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { // if there are attributes available, set them in our bitmap and increment // the number of fields. - if len(attributes) > 0 { + if !p.bm.contains(10) && len(attributes) > 0 { tc.attributes = attributes p.bm.set(10) p.fields += 1 } + p.chunks = append(p.chunks, tc) p.recordItem() return p.stats(), err @@ -253,6 +258,7 @@ func (p *payloadV1) Read(b []byte) (n int, err error) { // encode writes existing payload fields into the buffer in msgp format. func (p *payloadV1) encode() { st := newStringTable() + p.buf = msgp.AppendMapHeader(p.buf, p.fields) // number of fields in payload p.buf = encodeField(p.buf, p.bm, 2, p.containerID, st) p.buf = encodeField(p.buf, p.bm, 3, p.languageName, st) p.buf = encodeField(p.buf, p.bm, 4, p.languageVersion, st) @@ -297,7 +303,7 @@ func encodeField[F fieldValue](buf []byte, bm bitmap, fieldID uint32, a F, st *s case float64: buf = msgp.AppendFloat64(buf, value) case int32, int64: - buf = msgp.AppendInt64(buf, value.(int64)) + buf = msgp.AppendInt64(buf, handleIntValue(value)) case []byte: buf = msgp.AppendBytes(buf, value) case arrayValue: @@ -481,7 +487,7 @@ func (p *payloadV1) encodeSpanEvents(fieldID int, spanEvents []spanEvent, st *st case spanEventAttributeTypeInt: attr[k] = anyValue{ valueType: IntValueType, - value: v.IntValue, + value: handleIntValue(v.IntValue), } case spanEventAttributeTypeDouble: attr[k] = anyValue{ @@ -673,8 +679,8 @@ func buildAnyValue(v any) *anyValue { return &anyValue{valueType: BoolValueType, value: v} case float64: return &anyValue{valueType: FloatValueType, value: v} - case int64, int32: - return &anyValue{valueType: IntValueType, value: v} + case int32, int64: + return &anyValue{valueType: IntValueType, value: handleIntValue(v)} case []byte: return &anyValue{valueType: BytesValueType, value: v} case arrayValue: @@ -706,6 +712,19 @@ func (a anyValue) encode(buf []byte) []byte { return buf } +// translate any int value to int64 +func handleIntValue(a any) int64 { + switch v := a.(type) { + case int64: + return v + case int32: + return int64(v) + default: + // Fallback for other integer types + return v.(int64) + } +} + type arrayValue []anyValue // keeps track of which fields have been set in the payload, with a @@ -973,12 +992,12 @@ func (span *Span) decode(b []byte, st *stringTable) ([]byte, error) { // read msgp string value switch idx { case 1: - span.name, o, err = msgp.ReadStringBytes(o) + span.service, o, err = msgp.ReadStringBytes(o) if err != nil { return o, err } case 2: - span.service, o, err = msgp.ReadStringBytes(o) + span.name, o, err = msgp.ReadStringBytes(o) if err != nil { return o, err } @@ -988,72 +1007,69 @@ func (span *Span) decode(b []byte, st *stringTable) ([]byte, error) { return o, err } case 4: - span.spanType, o, err = msgp.ReadStringBytes(o) + span.spanID, o, err = msgp.ReadUint64Bytes(o) if err != nil { return o, err } case 5: - span.start, o, err = msgp.ReadInt64Bytes(o) + span.parentID, o, err = msgp.ReadUint64Bytes(o) if err != nil { return o, err } case 6: + span.start, o, err = msgp.ReadInt64Bytes(o) + if err != nil { + return o, err + } + case 7: span.duration, o, err = msgp.ReadInt64Bytes(o) if err != nil { return o, err } - // case 7: - // span.meta, o, err = DecodeKeyValueList(o, st) - // if err != nil { - // return o, err - // } - // case 8: - // span.metaStruct, o, err = DecodeKeyValueList(o, st) - // if err != nil { - // return o, err - // } + case 8: + var v bool + v, o, err = msgp.ReadBoolBytes(o) + if err != nil { + return o, err + } + if v { + span.error = 1 + } else { + span.error = 0 + } // case 9: - // span.metrics, o, err = DecodeKeyValueList(o, st) + // span.attributes, o, err = DecodeKeyValueList(o, st) // if err != nil { // return o, err // } case 10: - span.spanID, o, err = msgp.ReadUint64Bytes(o) + span.spanType, o, err = msgp.ReadStringBytes(o) if err != nil { return o, err } case 11: - span.traceID, o, err = msgp.ReadUint64Bytes(o) + span.spanLinks, o, err = DecodeSpanLinks(o, st) if err != nil { return o, err } case 12: - span.parentID, o, err = msgp.ReadUint64Bytes(o) + span.spanEvents, o, err = DecodeSpanEvents(o, st) if err != nil { return o, err } case 13: - var v bool - v, o, err = msgp.ReadBoolBytes(o) - if err != nil { - return o, err - } - if v { - span.error = 1 - } else { - span.error = 0 + var env string + env, o, err = msgp.ReadStringBytes(o) + if env != "" && err != nil { + span.SetTag(ext.Environment, env) } case 14: - span.spanLinks, o, err = DecodeSpanLinks(o, st) - if err != nil { - return o, err + var ver string + ver, o, err = msgp.ReadStringBytes(o) + if ver != "" && err != nil { + span.SetTag(ext.Version, ver) } case 15: - span.spanEvents, o, err = DecodeSpanEvents(o, st) - if err != nil { - return o, err - } - case 16: span.integration, o, err = msgp.ReadStringBytes(o) if err != nil { return o, err @@ -1210,7 +1226,8 @@ func decodeAnyValue(b []byte, strings *stringTable) (anyValue, []byte, error) { if err != nil { return anyValue{}, o, err } - return anyValue{valueType: IntValueType, value: i}, o, nil + intVal := handleIntValue(i) + return anyValue{valueType: IntValueType, value: intVal}, o, nil case BytesValueType: b, o, err := msgp.ReadBytesBytes(o, nil) if err != nil { From dc26c98a3cc3c96fc9275f75ac3d6411c450d842 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dario=20Casta=C3=B1=C3=A9?= Date: Thu, 9 Oct 2025 12:40:08 +0200 Subject: [PATCH 31/75] fix(ddtrace/tracer): don't encode map header twice --- ddtrace/tracer/payload_v1.go | 1 - 1 file changed, 1 deletion(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 05f0a4d94a..46dc5ab3f0 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -258,7 +258,6 @@ func (p *payloadV1) Read(b []byte) (n int, err error) { // encode writes existing payload fields into the buffer in msgp format. func (p *payloadV1) encode() { st := newStringTable() - p.buf = msgp.AppendMapHeader(p.buf, p.fields) // number of fields in payload p.buf = encodeField(p.buf, p.bm, 2, p.containerID, st) p.buf = encodeField(p.buf, p.bm, 3, p.languageName, st) p.buf = encodeField(p.buf, p.bm, 4, p.languageVersion, st) From 90309255a58f48c6161be76c540376d8c1abf6f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dario=20Casta=C3=B1=C3=A9?= Date: Thu, 9 Oct 2025 14:24:14 +0200 Subject: [PATCH 32/75] fix(ddtrace/tracer): use string table to decode strings --- ddtrace/tracer/payload_v1.go | 75 ++++++++++++++++++++++-------------- 1 file changed, 47 insertions(+), 28 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 46dc5ab3f0..86492df377 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -8,6 +8,7 @@ package tracer import ( "bytes" "encoding/binary" + "errors" "fmt" "strconv" "sync/atomic" @@ -892,7 +893,7 @@ func DecodeTraceChunks(b []byte, st *stringTable) ([]traceChunk, []byte, error) } for range numChunks { tc := traceChunk{} - b, err = tc.decode(b, st) + o, err = tc.decode(o, st) if err != nil { return nil, o, err } @@ -908,7 +909,10 @@ func (tc *traceChunk) decode(b []byte, st *stringTable) ([]byte, error) { } for range numFields { // read msgp field ID - var idx uint32 + var ( + idx uint32 + ok bool + ) idx, o, err = msgp.ReadUint32Bytes(o) if err != nil { return o, err @@ -922,9 +926,9 @@ func (tc *traceChunk) decode(b []byte, st *stringTable) ([]byte, error) { return o, err } case 2: - tc.origin, o, err = msgp.ReadStringBytes(o) - if err != nil { - return o, err + tc.origin, o, ok = st.Read(o) + if !ok { + return o, unableDecodeStringError } case 3: tc.attributes, o, err = DecodeKeyValueList(o, st) @@ -966,7 +970,7 @@ func DecodeSpans(b []byte, st *stringTable) (spanList, []byte, error) { } for range numSpans { span := Span{} - b, err = span.decode(b, st) + o, err = span.decode(o, st) if err != nil { return nil, o, err } @@ -981,8 +985,10 @@ func (span *Span) decode(b []byte, st *stringTable) ([]byte, error) { return o, err } for range numFields { - // read msgp field ID - var idx uint32 + var ( + idx uint32 // read msgp field ID + ok bool + ) idx, o, err = msgp.ReadUint32Bytes(o) if err != nil { return o, err @@ -991,19 +997,19 @@ func (span *Span) decode(b []byte, st *stringTable) ([]byte, error) { // read msgp string value switch idx { case 1: - span.service, o, err = msgp.ReadStringBytes(o) - if err != nil { - return o, err + span.service, o, ok = st.Read(o) + if !ok { + return o, unableDecodeStringError } case 2: - span.name, o, err = msgp.ReadStringBytes(o) - if err != nil { - return o, err + span.name, o, ok = st.Read(o) + if !ok { + return o, unableDecodeStringError } case 3: - span.resource, o, err = msgp.ReadStringBytes(o) - if err != nil { - return o, err + span.resource, o, ok = st.Read(o) + if !ok { + return o, unableDecodeStringError } case 4: span.spanID, o, err = msgp.ReadUint64Bytes(o) @@ -1042,9 +1048,9 @@ func (span *Span) decode(b []byte, st *stringTable) ([]byte, error) { // return o, err // } case 10: - span.spanType, o, err = msgp.ReadStringBytes(o) - if err != nil { - return o, err + span.spanType, o, ok = st.Read(o) + if !ok { + return o, unableDecodeStringError } case 11: span.spanLinks, o, err = DecodeSpanLinks(o, st) @@ -1058,21 +1064,32 @@ func (span *Span) decode(b []byte, st *stringTable) ([]byte, error) { } case 13: var env string - env, o, err = msgp.ReadStringBytes(o) - if env != "" && err != nil { + env, o, ok = st.Read(o) + if !ok { + return o, unableDecodeStringError + } + if env != "" { span.SetTag(ext.Environment, env) } case 14: var ver string - ver, o, err = msgp.ReadStringBytes(o) - if ver != "" && err != nil { + ver, o, ok = st.Read(o) + if !ok { + return o, unableDecodeStringError + } + if ver != "" { span.SetTag(ext.Version, ver) } case 15: - span.integration, o, err = msgp.ReadStringBytes(o) - if err != nil { - return o, err + var component string + component, o, ok = st.Read(o) + if !ok { + return o, unableDecodeStringError } + if component != "" { + span.SetTag(ext.Component, component) + } + // TODO(darccio): otel.SpanKind kind = 16 is missing. default: return o, fmt.Errorf("unexpected field ID %d", idx) } @@ -1196,6 +1213,8 @@ func (event *spanEvent) decode(b []byte, st *stringTable) ([]byte, error) { return o, err } +var unableDecodeStringError = errors.New("unable to read string value") + func decodeAnyValue(b []byte, strings *stringTable) (anyValue, []byte, error) { vType, o, err := msgp.ReadInt32Bytes(b) if err != nil { @@ -1205,7 +1224,7 @@ func decodeAnyValue(b []byte, strings *stringTable) (anyValue, []byte, error) { case StringValueType: str, o, ok := strings.Read(o) if !ok { - return anyValue{}, o, fmt.Errorf("unable to read string value") + return anyValue{}, o, unableDecodeStringError } return anyValue{valueType: StringValueType, value: str}, o, nil case BoolValueType: From 269988e134e45d812bd679f7537743270e063c32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dario=20Casta=C3=B1=C3=A9?= Date: Thu, 9 Oct 2025 14:44:20 +0200 Subject: [PATCH 33/75] fix(ddtrace/tracer): encode uint32 and uint64 field values --- ddtrace/tracer/payload_v1.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 86492df377..af60db9fdc 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -304,6 +304,10 @@ func encodeField[F fieldValue](buf []byte, bm bitmap, fieldID uint32, a F, st *s buf = msgp.AppendFloat64(buf, value) case int32, int64: buf = msgp.AppendInt64(buf, handleIntValue(value)) + case uint32: + buf = msgp.AppendUint32(buf, value) + case uint64: + buf = msgp.AppendUint64(buf, value) case []byte: buf = msgp.AppendBytes(buf, value) case arrayValue: @@ -393,11 +397,7 @@ func (p *payloadV1) encodeSpans(fieldID int, spans spanList, st *stringTable) er p.buf = encodeField(p.buf, fullSetBitmap, 5, span.parentID, st) p.buf = encodeField(p.buf, fullSetBitmap, 6, span.start, st) p.buf = encodeField(p.buf, fullSetBitmap, 7, span.duration, st) - if span.error != 0 { - p.buf = encodeField(p.buf, fullSetBitmap, 8, true, st) - } else { - p.buf = encodeField(p.buf, fullSetBitmap, 8, false, st) - } + p.buf = encodeField(p.buf, fullSetBitmap, 8, span.error != 0, st) // span attributes combine the meta (tags), metrics and meta_struct attr := map[string]anyValue{} From 8c7bef3b8e7fcfa42d9052952be2590f7e38d07d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dario=20Casta=C3=B1=C3=A9?= Date: Thu, 9 Oct 2025 17:10:32 +0200 Subject: [PATCH 34/75] fix(ddtrace/tracer): remove redundant err checks and return the right buffer on failure --- ddtrace/tracer/payload_v1.go | 95 ++++++++++++------------------------ 1 file changed, 31 insertions(+), 64 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index af60db9fdc..aa17fffbf7 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -577,7 +577,7 @@ func (p *payloadV1) SetAppVersion(v string) { func (p *payloadV1) decodeBuffer() ([]byte, error) { numFields, o, err := msgp.ReadMapHeaderBytes(p.buf) if err != nil { - return o, err + return p.buf, err } p.buf = o p.fields = numFields @@ -757,7 +757,7 @@ func (i index) encode(buf []byte) []byte { func (i *index) decode(buf []byte) ([]byte, error) { val, o, err := msgp.ReadUint32Bytes(buf) if err != nil { - return o, err + return buf, err } *i = index(val) return o, nil @@ -773,7 +773,7 @@ func (s stringValue) encode(buf []byte) []byte { func (s *stringValue) decode(buf []byte) ([]byte, error) { val, o, err := msgp.ReadStringBytes(buf) if err != nil { - return o, err + return buf, err } *s = stringValue(val) return o, nil @@ -821,7 +821,7 @@ func (s *stringTable) Read(b []byte) (string, []byte, bool) { var sv stringValue o, err := sv.decode(b) if err != nil { - return "", o, false + return "", b, false } str := string(sv) s.Add(str) @@ -831,7 +831,7 @@ func (s *stringTable) Read(b []byte) (string, []byte, bool) { var i index o, err := i.decode(b) if err != nil { - return "", o, false + return "", b, false } return s.strings[i], o, true } @@ -889,7 +889,7 @@ func DecodeTraceChunks(b []byte, st *stringTable) ([]traceChunk, []byte, error) out := []traceChunk{} numChunks, o, err := msgp.ReadArrayHeaderBytes(b) if err != nil { - return nil, o, err + return nil, b, err } for range numChunks { tc := traceChunk{} @@ -904,10 +904,10 @@ func DecodeTraceChunks(b []byte, st *stringTable) ([]traceChunk, []byte, error) func (tc *traceChunk) decode(b []byte, st *stringTable) ([]byte, error) { numFields, o, err := msgp.ReadMapHeaderBytes(b) - if err != nil { - return o, err - } for range numFields { + if err != nil { + return b, err + } // read msgp field ID var ( idx uint32 @@ -922,39 +922,21 @@ func (tc *traceChunk) decode(b []byte, st *stringTable) ([]byte, error) { switch idx { case 1: tc.priority, o, err = msgp.ReadInt32Bytes(o) - if err != nil { - return o, err - } case 2: tc.origin, o, ok = st.Read(o) if !ok { - return o, unableDecodeStringError + err = unableDecodeStringError } case 3: tc.attributes, o, err = DecodeKeyValueList(o, st) - if err != nil { - return o, err - } case 4: tc.spans, o, err = DecodeSpans(o, st) - if err != nil { - return o, err - } case 5: tc.droppedTrace, o, err = msgp.ReadBoolBytes(o) - if err != nil { - return o, err - } case 6: tc.traceID, o, err = msgp.ReadBytesBytes(o, nil) - if err != nil { - return o, err - } case 7: tc.samplingMechanism, o, err = msgp.ReadUint32Bytes(o) - if err != nil { - return o, err - } default: return o, fmt.Errorf("unexpected field ID %d", idx) } @@ -966,7 +948,7 @@ func DecodeSpans(b []byte, st *stringTable) (spanList, []byte, error) { out := spanList{} numSpans, o, err := msgp.ReadArrayHeaderBytes(b) if err != nil { - return nil, o, err + return nil, b, err } for range numSpans { span := Span{} @@ -981,10 +963,10 @@ func DecodeSpans(b []byte, st *stringTable) (spanList, []byte, error) { func (span *Span) decode(b []byte, st *stringTable) ([]byte, error) { numFields, o, err := msgp.ReadMapHeaderBytes(b) - if err != nil { - return o, err - } for range numFields { + if err != nil { + return b, err + } var ( idx uint32 // read msgp field ID ok bool @@ -999,43 +981,31 @@ func (span *Span) decode(b []byte, st *stringTable) ([]byte, error) { case 1: span.service, o, ok = st.Read(o) if !ok { - return o, unableDecodeStringError + err = unableDecodeStringError } case 2: span.name, o, ok = st.Read(o) if !ok { - return o, unableDecodeStringError + err = unableDecodeStringError } case 3: span.resource, o, ok = st.Read(o) if !ok { - return o, unableDecodeStringError + err = unableDecodeStringError } case 4: span.spanID, o, err = msgp.ReadUint64Bytes(o) - if err != nil { - return o, err - } case 5: span.parentID, o, err = msgp.ReadUint64Bytes(o) - if err != nil { - return o, err - } case 6: span.start, o, err = msgp.ReadInt64Bytes(o) - if err != nil { - return o, err - } case 7: span.duration, o, err = msgp.ReadInt64Bytes(o) - if err != nil { - return o, err - } case 8: var v bool v, o, err = msgp.ReadBoolBytes(o) if err != nil { - return o, err + break } if v { span.error = 1 @@ -1050,23 +1020,18 @@ func (span *Span) decode(b []byte, st *stringTable) ([]byte, error) { case 10: span.spanType, o, ok = st.Read(o) if !ok { - return o, unableDecodeStringError + err = unableDecodeStringError } case 11: span.spanLinks, o, err = DecodeSpanLinks(o, st) - if err != nil { - return o, err - } case 12: span.spanEvents, o, err = DecodeSpanEvents(o, st) - if err != nil { - return o, err - } case 13: var env string env, o, ok = st.Read(o) if !ok { - return o, unableDecodeStringError + err = unableDecodeStringError + break } if env != "" { span.SetTag(ext.Environment, env) @@ -1075,7 +1040,8 @@ func (span *Span) decode(b []byte, st *stringTable) ([]byte, error) { var ver string ver, o, ok = st.Read(o) if !ok { - return o, unableDecodeStringError + err = unableDecodeStringError + break } if ver != "" { span.SetTag(ext.Version, ver) @@ -1084,7 +1050,8 @@ func (span *Span) decode(b []byte, st *stringTable) ([]byte, error) { var component string component, o, ok = st.Read(o) if !ok { - return o, unableDecodeStringError + err = unableDecodeStringError + break } if component != "" { span.SetTag(ext.Component, component) @@ -1101,7 +1068,7 @@ func DecodeSpanLinks(b []byte, st *stringTable) ([]SpanLink, []byte, error) { out := []SpanLink{} numLinks, o, err := msgp.ReadArrayHeaderBytes(b) if err != nil { - return nil, o, err + return nil, b, err } for range numLinks { link := SpanLink{} @@ -1117,7 +1084,7 @@ func DecodeSpanLinks(b []byte, st *stringTable) ([]SpanLink, []byte, error) { func (link *SpanLink) decode(b []byte, st *stringTable) ([]byte, error) { numFields, o, err := msgp.ReadMapHeaderBytes(b) if err != nil { - return o, err + return b, err } for range numFields { // read msgp field ID @@ -1165,7 +1132,7 @@ func DecodeSpanEvents(b []byte, st *stringTable) ([]spanEvent, []byte, error) { out := []spanEvent{} numEvents, o, err := msgp.ReadArrayHeaderBytes(b) if err != nil { - return nil, o, err + return nil, b, err } for range numEvents { event := spanEvent{} @@ -1181,7 +1148,7 @@ func DecodeSpanEvents(b []byte, st *stringTable) ([]spanEvent, []byte, error) { func (event *spanEvent) decode(b []byte, st *stringTable) ([]byte, error) { numFields, o, err := msgp.ReadMapHeaderBytes(b) if err != nil { - return o, err + return b, err } for range numFields { // read msgp field ID @@ -1218,7 +1185,7 @@ var unableDecodeStringError = errors.New("unable to read string value") func decodeAnyValue(b []byte, strings *stringTable) (anyValue, []byte, error) { vType, o, err := msgp.ReadInt32Bytes(b) if err != nil { - return anyValue{}, o, err + return anyValue{}, b, err } switch vType { case StringValueType: @@ -1279,7 +1246,7 @@ func decodeAnyValue(b []byte, strings *stringTable) (anyValue, []byte, error) { func DecodeKeyValueList(b []byte, strings *stringTable) (map[string]anyValue, []byte, error) { numFields, o, err := msgp.ReadMapHeaderBytes(b) if err != nil { - return nil, o, err + return nil, b, err } kv := map[string]anyValue{} From 8f06c7bcb8663101744f6b792684e9c689c0f017 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dario=20Casta=C3=B1=C3=A9?= Date: Thu, 9 Oct 2025 17:11:16 +0200 Subject: [PATCH 35/75] fix(ddtrace/tracer): rename error variable to comply with linter --- ddtrace/tracer/payload_v1.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index aa17fffbf7..7ce9a64e9e 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -925,7 +925,7 @@ func (tc *traceChunk) decode(b []byte, st *stringTable) ([]byte, error) { case 2: tc.origin, o, ok = st.Read(o) if !ok { - err = unableDecodeStringError + err = errUnableDecodeString } case 3: tc.attributes, o, err = DecodeKeyValueList(o, st) @@ -981,17 +981,17 @@ func (span *Span) decode(b []byte, st *stringTable) ([]byte, error) { case 1: span.service, o, ok = st.Read(o) if !ok { - err = unableDecodeStringError + err = errUnableDecodeString } case 2: span.name, o, ok = st.Read(o) if !ok { - err = unableDecodeStringError + err = errUnableDecodeString } case 3: span.resource, o, ok = st.Read(o) if !ok { - err = unableDecodeStringError + err = errUnableDecodeString } case 4: span.spanID, o, err = msgp.ReadUint64Bytes(o) @@ -1020,7 +1020,7 @@ func (span *Span) decode(b []byte, st *stringTable) ([]byte, error) { case 10: span.spanType, o, ok = st.Read(o) if !ok { - err = unableDecodeStringError + err = errUnableDecodeString } case 11: span.spanLinks, o, err = DecodeSpanLinks(o, st) @@ -1030,7 +1030,7 @@ func (span *Span) decode(b []byte, st *stringTable) ([]byte, error) { var env string env, o, ok = st.Read(o) if !ok { - err = unableDecodeStringError + err = errUnableDecodeString break } if env != "" { @@ -1040,7 +1040,7 @@ func (span *Span) decode(b []byte, st *stringTable) ([]byte, error) { var ver string ver, o, ok = st.Read(o) if !ok { - err = unableDecodeStringError + err = errUnableDecodeString break } if ver != "" { @@ -1050,7 +1050,7 @@ func (span *Span) decode(b []byte, st *stringTable) ([]byte, error) { var component string component, o, ok = st.Read(o) if !ok { - err = unableDecodeStringError + err = errUnableDecodeString break } if component != "" { @@ -1180,7 +1180,7 @@ func (event *spanEvent) decode(b []byte, st *stringTable) ([]byte, error) { return o, err } -var unableDecodeStringError = errors.New("unable to read string value") +var errUnableDecodeString = errors.New("unable to read string value") func decodeAnyValue(b []byte, strings *stringTable) (anyValue, []byte, error) { vType, o, err := msgp.ReadInt32Bytes(b) @@ -1191,7 +1191,7 @@ func decodeAnyValue(b []byte, strings *stringTable) (anyValue, []byte, error) { case StringValueType: str, o, ok := strings.Read(o) if !ok { - return anyValue{}, o, unableDecodeStringError + return anyValue{}, o, errUnableDecodeString } return anyValue{valueType: StringValueType, value: str}, o, nil case BoolValueType: From 80dbd5491272196cc3e96f0094ceb777301f3c96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dario=20Casta=C3=B1=C3=A9?= Date: Thu, 9 Oct 2025 17:35:18 +0200 Subject: [PATCH 36/75] fix(ddtrace/tracer): don't use receiver's bitmap when encoding attributes, spans, etc. --- ddtrace/tracer/payload_v1.go | 66 +++++++++++++++++------------------- 1 file changed, 31 insertions(+), 35 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 7ce9a64e9e..5799c7dfb3 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -268,13 +268,9 @@ func (p *payloadV1) encode() { p.buf = encodeField(p.buf, p.bm, 8, p.hostname, st) p.buf = encodeField(p.buf, p.bm, 9, p.appVersion, st) - if len(p.attributes) > 0 { - p.encodeAttributes(10, p.attributes, st) - } + p.encodeAttributes(p.bm, 10, p.attributes, st) - if len(p.chunks) > 0 { - p.encodeTraceChunks(11, p.chunks, st) - } + p.encodeTraceChunks(p.bm, 11, p.chunks, st) } type fieldValue interface { @@ -319,9 +315,9 @@ func encodeField[F fieldValue](buf []byte, bm bitmap, fieldID uint32, a F, st *s return buf } -func (p *payloadV1) encodeAttributes(fieldID int, kv map[string]anyValue, st *stringTable) error { - if !p.bm.contains(uint32(fieldID)) || len(kv) == 0 { - return nil +func (p *payloadV1) encodeAttributes(bm bitmap, fieldID int, kv map[string]anyValue, st *stringTable) (bool, error) { + if !bm.contains(uint32(fieldID)) || len(kv) == 0 { + return false, nil } p.buf = msgp.AppendUint32(p.buf, uint32(fieldID)) // msgp key @@ -338,12 +334,12 @@ func (p *payloadV1) encodeAttributes(fieldID int, kv map[string]anyValue, st *st // encode value p.buf = v.encode(p.buf) } - return nil + return true, nil } -func (p *payloadV1) encodeTraceChunks(fieldID int, tc []traceChunk, st *stringTable) error { - if len(tc) == 0 || !p.bm.contains(uint32(fieldID)) { - return nil +func (p *payloadV1) encodeTraceChunks(bm bitmap, fieldID int, tc []traceChunk, st *stringTable) (bool, error) { + if len(tc) == 0 || !bm.contains(uint32(fieldID)) { + return false, nil } p.buf = msgp.AppendUint32(p.buf, uint32(fieldID)) // msgp key @@ -358,10 +354,10 @@ func (p *payloadV1) encodeTraceChunks(fieldID int, tc []traceChunk, st *stringTa p.buf = encodeField(p.buf, fullSetBitmap, 2, chunk.origin, st) // attributes - p.encodeAttributes(3, chunk.attributes, st) + p.encodeAttributes(fullSetBitmap, 3, chunk.attributes, st) // spans - p.encodeSpans(4, chunk.spans, st) + p.encodeSpans(fullSetBitmap, 4, chunk.spans, st) // droppedTrace p.buf = encodeField(p.buf, fullSetBitmap, 5, chunk.droppedTrace, st) @@ -373,12 +369,12 @@ func (p *payloadV1) encodeTraceChunks(fieldID int, tc []traceChunk, st *stringTa p.buf = encodeField(p.buf, fullSetBitmap, 7, chunk.samplingMechanism, st) } - return nil + return true, nil } -func (p *payloadV1) encodeSpans(fieldID int, spans spanList, st *stringTable) error { - if len(spans) == 0 || !p.bm.contains(uint32(fieldID)) { - return nil +func (p *payloadV1) encodeSpans(bm bitmap, fieldID int, spans spanList, st *stringTable) (bool, error) { + if len(spans) == 0 || !bm.contains(uint32(fieldID)) { + return false, nil } p.buf = msgp.AppendUint32(p.buf, uint32(fieldID)) // msgp key @@ -388,7 +384,7 @@ func (p *payloadV1) encodeSpans(fieldID int, spans spanList, st *stringTable) er if span == nil { continue } - p.buf = msgp.AppendMapHeader(p.buf, 16) // number of fields in span + p.buf = msgp.AppendMapHeader(p.buf, 15) // number of fields in span p.buf = encodeField(p.buf, fullSetBitmap, 1, span.service, st) p.buf = encodeField(p.buf, fullSetBitmap, 2, span.name, st) @@ -419,11 +415,11 @@ func (p *payloadV1) encodeSpans(fieldID int, spans spanList, st *stringTable) er attr[k] = *av } } - p.encodeAttributes(9, attr, st) + p.encodeAttributes(fullSetBitmap, 9, attr, st) p.buf = encodeField(p.buf, fullSetBitmap, 10, span.spanType, st) - p.encodeSpanLinks(11, span.spanLinks, st) - p.encodeSpanEvents(12, span.spanEvents, st) + p.encodeSpanLinks(fullSetBitmap, 11, span.spanLinks, st) + p.encodeSpanEvents(fullSetBitmap, 12, span.spanEvents, st) env := span.meta[ext.Environment] p.buf = encodeField(p.buf, fullSetBitmap, 13, env, st) @@ -433,12 +429,12 @@ func (p *payloadV1) encodeSpans(fieldID int, spans spanList, st *stringTable) er p.buf = encodeField(p.buf, fullSetBitmap, 15, span.integration, st) } - return nil + return true, nil } -func (p *payloadV1) encodeSpanLinks(fieldID int, spanLinks []SpanLink, st *stringTable) error { - if len(spanLinks) == 0 || !p.bm.contains(uint32(fieldID)) { - return nil +func (p *payloadV1) encodeSpanLinks(bm bitmap, fieldID int, spanLinks []SpanLink, st *stringTable) (bool, error) { + if len(spanLinks) == 0 || !bm.contains(uint32(fieldID)) { + return false, nil } p.buf = msgp.AppendUint32(p.buf, uint32(fieldID)) // msgp key p.buf = msgp.AppendArrayHeader(p.buf, uint32(len(spanLinks))) // number of span links @@ -458,14 +454,14 @@ func (p *payloadV1) encodeSpanLinks(fieldID int, spanLinks []SpanLink, st *strin value: stringValue(v), } } - p.encodeAttributes(3, attr, st) + p.encodeAttributes(fullSetBitmap, 3, attr, st) } - return nil + return true, nil } -func (p *payloadV1) encodeSpanEvents(fieldID int, spanEvents []spanEvent, st *stringTable) error { - if len(spanEvents) == 0 || !p.bm.contains(uint32(fieldID)) { - return nil +func (p *payloadV1) encodeSpanEvents(bm bitmap, fieldID int, spanEvents []spanEvent, st *stringTable) (bool, error) { + if len(spanEvents) == 0 || !bm.contains(uint32(fieldID)) { + return false, nil } p.buf = msgp.AppendUint32(p.buf, uint32(fieldID)) // msgp key p.buf = msgp.AppendArrayHeader(p.buf, uint32(len(spanEvents))) // number of span events @@ -508,9 +504,9 @@ func (p *payloadV1) encodeSpanEvents(fieldID int, spanEvents []spanEvent, st *st log.Warn("dropped unsupported span event attribute type %d", v.Type) } } - p.encodeAttributes(3, attr, st) + p.encodeAttributes(fullSetBitmap, 3, attr, st) } - return nil + return true, nil } // Getters for payloadV1 fields @@ -963,7 +959,7 @@ func DecodeSpans(b []byte, st *stringTable) (spanList, []byte, error) { func (span *Span) decode(b []byte, st *stringTable) ([]byte, error) { numFields, o, err := msgp.ReadMapHeaderBytes(b) - for range numFields { + for range numFields - 1 { if err != nil { return b, err } From b64c68f94e35d3d72d8534d8b80bb3d25152fad2 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Thu, 9 Oct 2025 15:44:01 -0400 Subject: [PATCH 37/75] early return on fail while encoding, also fix some types --- ddtrace/tracer/payload_v1.go | 134 ++++++++++++++++++++++------------- 1 file changed, 84 insertions(+), 50 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 5799c7dfb3..2c85a9ef3e 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -316,12 +316,13 @@ func encodeField[F fieldValue](buf []byte, bm bitmap, fieldID uint32, a F, st *s } func (p *payloadV1) encodeAttributes(bm bitmap, fieldID int, kv map[string]anyValue, st *stringTable) (bool, error) { - if !bm.contains(uint32(fieldID)) || len(kv) == 0 { + if !bm.contains(uint32(fieldID)) { return false, nil } p.buf = msgp.AppendUint32(p.buf, uint32(fieldID)) // msgp key p.buf = msgp.AppendMapHeader(p.buf, uint32(len(kv))) // number of item pairs in map + for k, v := range kv { // encode msgp key if idx, ok := st.Get(string(k)); ok { @@ -384,7 +385,8 @@ func (p *payloadV1) encodeSpans(bm bitmap, fieldID int, spans spanList, st *stri if span == nil { continue } - p.buf = msgp.AppendMapHeader(p.buf, 15) // number of fields in span + // In encodeSpans function, after line 388: + p.buf = msgp.AppendMapHeader(p.buf, 16) // number of fields in span p.buf = encodeField(p.buf, fullSetBitmap, 1, span.service, st) p.buf = encodeField(p.buf, fullSetBitmap, 2, span.name, st) @@ -400,7 +402,7 @@ func (p *payloadV1) encodeSpans(bm bitmap, fieldID int, spans spanList, st *stri for k, v := range span.meta { attr[k] = anyValue{ valueType: StringValueType, - value: stringValue(v), + value: v, } } for k, v := range span.metrics { @@ -427,13 +429,18 @@ func (p *payloadV1) encodeSpans(bm bitmap, fieldID int, spans spanList, st *stri version := span.meta[ext.Version] p.buf = encodeField(p.buf, fullSetBitmap, 14, version, st) - p.buf = encodeField(p.buf, fullSetBitmap, 15, span.integration, st) + component := span.meta[ext.Component] + p.buf = encodeField(p.buf, fullSetBitmap, 15, component, st) + + // And especially field 16: + spanKind := span.meta[ext.SpanKind] + p.buf = encodeField(p.buf, fullSetBitmap, 16, spanKind, st) } return true, nil } func (p *payloadV1) encodeSpanLinks(bm bitmap, fieldID int, spanLinks []SpanLink, st *stringTable) (bool, error) { - if len(spanLinks) == 0 || !bm.contains(uint32(fieldID)) { + if !bm.contains(uint32(fieldID)) { return false, nil } p.buf = msgp.AppendUint32(p.buf, uint32(fieldID)) // msgp key @@ -451,7 +458,7 @@ func (p *payloadV1) encodeSpanLinks(bm bitmap, fieldID int, spanLinks []SpanLink for k, v := range link.Attributes { attr[k] = anyValue{ valueType: StringValueType, - value: stringValue(v), + value: v, } } p.encodeAttributes(fullSetBitmap, 3, attr, st) @@ -460,7 +467,7 @@ func (p *payloadV1) encodeSpanLinks(bm bitmap, fieldID int, spanLinks []SpanLink } func (p *payloadV1) encodeSpanEvents(bm bitmap, fieldID int, spanEvents []spanEvent, st *stringTable) (bool, error) { - if len(spanEvents) == 0 || !bm.contains(uint32(fieldID)) { + if !bm.contains(uint32(fieldID)) { return false, nil } p.buf = msgp.AppendUint32(p.buf, uint32(fieldID)) // msgp key @@ -581,7 +588,11 @@ func (p *payloadV1) decodeBuffer() ([]byte, error) { p.updateHeader() st := newStringTable() + fieldCount := 1 for { + if len(o) == 0 || err != nil { + break + } // read msgp field ID var idx uint32 idx, o, err = msgp.ReadUint32Bytes(o) @@ -636,9 +647,7 @@ func (p *payloadV1) decodeBuffer() ([]byte, error) { default: err = fmt.Errorf("unexpected field ID %d", idx) } - if len(o) == 0 || err != nil { - break - } + fieldCount++ } return o, err } @@ -690,7 +699,8 @@ func (a anyValue) encode(buf []byte) []byte { buf = msgp.AppendInt32(buf, int32(a.valueType)) switch a.valueType { case StringValueType: - buf = a.value.(stringValue).encode(buf) + s := a.value.(string) + buf = stringValue(s).encode(buf) case BoolValueType: buf = msgp.AppendBool(buf, a.value.(bool)) case FloatValueType: @@ -702,7 +712,7 @@ func (a anyValue) encode(buf []byte) []byte { case ArrayValueType: buf = msgp.AppendArrayHeader(buf, uint32(len(a.value.(arrayValue)))) for _, v := range a.value.(arrayValue) { - v.encode(buf) + buf = v.encode(buf) } } return buf @@ -725,19 +735,19 @@ type arrayValue []anyValue // keeps track of which fields have been set in the payload, with a // 1 for represented fields and 0 for unset fields. -type bitmap int16 +type bitmap int32 var fullSetBitmap bitmap = -1 func (b *bitmap) set(bit uint32) { - if bit >= 16 { + if bit >= 32 { return } *b |= 1 << bit } func (b bitmap) contains(bit uint32) bool { - if bit >= 16 { + if bit >= 32 { return false } return b&(1< Date: Thu, 9 Oct 2025 16:58:15 -0400 Subject: [PATCH 38/75] use string table during encoding and add more int coverage --- ddtrace/tracer/payload_v1.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 2c85a9ef3e..19111f767a 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -333,7 +333,7 @@ func (p *payloadV1) encodeAttributes(bm bitmap, fieldID int, kv map[string]anyVa } // encode value - p.buf = v.encode(p.buf) + p.buf = v.encode(p.buf, st) } return true, nil } @@ -695,12 +695,19 @@ func buildAnyValue(v any) *anyValue { } } -func (a anyValue) encode(buf []byte) []byte { +func (a anyValue) encode(buf []byte, st *stringTable) []byte { buf = msgp.AppendInt32(buf, int32(a.valueType)) switch a.valueType { case StringValueType: s := a.value.(string) + if idx, ok := st.Get(s); ok { + fmt.Printf("DEBUG: Encoded STRING IDX %v\n", idx) + buf = idx.encode(buf) + } else { + fmt.Printf("DEBUG: Encoded STRING %v\n", s) buf = stringValue(s).encode(buf) + st.Add(s) + } case BoolValueType: buf = msgp.AppendBool(buf, a.value.(bool)) case FloatValueType: @@ -712,7 +719,7 @@ func (a anyValue) encode(buf []byte) []byte { case ArrayValueType: buf = msgp.AppendArrayHeader(buf, uint32(len(a.value.(arrayValue)))) for _, v := range a.value.(arrayValue) { - buf = v.encode(buf) + buf = v.encode(buf, st) } } return buf @@ -851,7 +858,7 @@ func getStreamingType(b byte) int { // String formats case 0xd9, 0xda, 0xdb: // str8, str16, str32 return 0 - case 0xce: // uint32 + case 0xcc, 0xcd, 0xce: // uint8, uint16, uint32 return 1 default: // Check for fixstr From e739a34671aff805cb7fd7b67851297f6d154512 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Thu, 9 Oct 2025 17:08:03 -0400 Subject: [PATCH 39/75] never overwrite buffers --- ddtrace/tracer/payload_v1.go | 41 ++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 19111f767a..f820826c72 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -309,7 +309,7 @@ func encodeField[F fieldValue](buf []byte, bm bitmap, fieldID uint32, a F, st *s case arrayValue: buf = msgp.AppendArrayHeader(buf, uint32(len(value))) for _, v := range value { - buf = v.encode(buf) + buf = v.encode(buf, st) } } return buf @@ -701,11 +701,9 @@ func (a anyValue) encode(buf []byte, st *stringTable) []byte { case StringValueType: s := a.value.(string) if idx, ok := st.Get(s); ok { - fmt.Printf("DEBUG: Encoded STRING IDX %v\n", idx) buf = idx.encode(buf) } else { - fmt.Printf("DEBUG: Encoded STRING %v\n", s) - buf = stringValue(s).encode(buf) + buf = stringValue(s).encode(buf) st.Add(s) } case BoolValueType: @@ -768,7 +766,7 @@ func (i index) encode(buf []byte) []byte { } func (i *index) decode(buf []byte) ([]byte, error) { - val, o, err := msgp.ReadUint32Bytes(buf) + val, o, err := msgp.ReadUintBytes(buf) if err != nil { return buf, err } @@ -1226,38 +1224,47 @@ func decodeAnyValue(b []byte, strings *stringTable) (anyValue, []byte, error) { } switch vType { case StringValueType: - str, o, ok := strings.Read(o) + var ( + str string + ok bool + ) + str, o, ok = strings.Read(o) if !ok { return anyValue{}, o, errUnableDecodeString } return anyValue{valueType: StringValueType, value: str}, o, nil case BoolValueType: - b, o, err := msgp.ReadBoolBytes(o) + var b bool + b, o, err = msgp.ReadBoolBytes(o) if err != nil { return anyValue{}, o, err } return anyValue{valueType: BoolValueType, value: b}, o, nil case FloatValueType: - f, o, err := msgp.ReadFloat64Bytes(o) + var f float64 + f, o, err = msgp.ReadFloat64Bytes(o) if err != nil { return anyValue{}, o, err } return anyValue{valueType: FloatValueType, value: f}, o, nil case IntValueType: - i, o, err := msgp.ReadInt64Bytes(o) + var i int64 + i, o, err = msgp.ReadInt64Bytes(o) if err != nil { return anyValue{}, o, err } intVal := handleIntValue(i) return anyValue{valueType: IntValueType, value: intVal}, o, nil case BytesValueType: - b, o, err := msgp.ReadBytesBytes(o, nil) + var b []byte + b, o, err = msgp.ReadBytesBytes(o, nil) if err != nil { return anyValue{}, o, err } return anyValue{valueType: BytesValueType, value: b}, o, nil case ArrayValueType: - len, o, err := msgp.ReadArrayHeaderBytes(o) + var len uint32 + len, o, err = msgp.ReadArrayHeaderBytes(o) if err != nil { return anyValue{}, o, err } @@ -1270,7 +1277,8 @@ func decodeAnyValue(b []byte, strings *stringTable) (anyValue, []byte, error) { } return anyValue{valueType: ArrayValueType, value: arrayValue}, o, nil case keyValueListType: - kv, o, err := DecodeKeyValueList(o, strings) + var kv map[string]anyValue + kv, o, err = DecodeKeyValueList(o, strings) if err != nil { return anyValue{}, o, err } @@ -1288,11 +1296,16 @@ func DecodeKeyValueList(b []byte, strings *stringTable) (map[string]anyValue, [] kv := map[string]anyValue{} for i := range numFields { - key, o, ok := strings.Read(o) + var ( + key string + ok bool + av anyValue + ) + key, o, ok = strings.Read(o) if !ok { return nil, o, fmt.Errorf("unable to read key of field %d", i) } - av, o, err := decodeAnyValue(o, strings) + av, o, err = decodeAnyValue(o, strings) if err != nil { return nil, o, err } From 3a6e7fa05b4ce41642de1240a16493a45538471f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dario=20Casta=C3=B1=C3=A9?= Date: Fri, 10 Oct 2025 15:16:44 +0200 Subject: [PATCH 40/75] fix(ddtrace/tracer): cache service as payload attribute; enhance tests --- ddtrace/tracer/payload_test.go | 9 +++++++-- ddtrace/tracer/payload_v1.go | 16 +++++++++------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/ddtrace/tracer/payload_test.go b/ddtrace/tracer/payload_test.go index 51e005d919..31c089276f 100644 --- a/ddtrace/tracer/payload_test.go +++ b/ddtrace/tracer/payload_test.go @@ -42,7 +42,7 @@ func newDetailedSpanList(n int) spanList { for i := 0; i < n; i++ { list[i] = newBasicSpan("span.list." + itoa[i%5+1]) list[i].start = fixedTime - list[i].service = "service." + itoa[i%5+1] + list[i].service = "golden" list[i].resource = "resource." + itoa[i%5+1] list[i].error = int32(i % 2) list[i].SetTag("tag."+itoa[i%5+1], "value."+itoa[i%5+1]) @@ -160,9 +160,14 @@ func TestPayloadV1Decode(t *testing.T) { _, err = buf.WriteTo(got) assert.NoError(err) - _, err = got.decodeBuffer() + o, err := got.decodeBuffer() assert.NoError(err) + assert.Empty(o) assert.NotEmpty(got.attributes) + assert.Equal(p.attributes, got.attributes) + assert.Equal(got.attributes["service"].value, "golden") + assert.Equal(p.chunks[0].traceID, got.chunks[0].traceID) + assert.Equal(p.chunks[0].spans[0].spanID, got.chunks[0].spans[0].spanID) }) } } diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index f820826c72..224802333a 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -115,18 +115,21 @@ func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { // For now, we blindly set the origin, priority, and attributes values for the chunk // In the future, attributes should hold values that are shared across all chunks in the payload - attributes := map[string]anyValue{} origin, priority, sm := "", 0, 0 for _, span := range t { if span == nil { break } + // If we haven't seen the service yet, we set it blindly assuming that all the spans created by + // a service must share the same value. + if _, ok := p.attributes["service"]; !ok { + p.attributes["service"] = anyValue{valueType: StringValueType, value: span.Root().service} + } if p, ok := span.Context().SamplingPriority(); ok { - origin = span.Context().origin - priority = p - attributes["service"] = anyValue{valueType: StringValueType, value: span.Root().service} + origin = span.Context().origin // TODO(darccio): are we sure that origin will be shared across all the spans in the chunk? + priority = p // TODO(darccio): the same goes for priority. dm := span.context.trace.propagatingTag(keyDecisionMaker) - sm, err = strconv.Atoi(dm) + sm, err = strconv.Atoi(dm) // TODO(darccio): shouldn't this string represent the absolute value of dm? if err != nil { log.Error("failed to convert decision maker to int: %s", err.Error()) } @@ -144,8 +147,7 @@ func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { // if there are attributes available, set them in our bitmap and increment // the number of fields. - if !p.bm.contains(10) && len(attributes) > 0 { - tc.attributes = attributes + if !p.bm.contains(10) && len(p.attributes) > 0 { p.bm.set(10) p.fields += 1 } From f3f86ab2fb7db7710e8221f9c33bd912b73f908d Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Fri, 10 Oct 2025 12:57:38 -0400 Subject: [PATCH 41/75] fix: make span link and event decode consistent with other functions --- ddtrace/tracer/payload_v1.go | 46 +++++++++++++++++------------------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 224802333a..892c5e68d0 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -118,7 +118,7 @@ func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { origin, priority, sm := "", 0, 0 for _, span := range t { if span == nil { - break + continue } // If we haven't seen the service yet, we set it blindly assuming that all the spans created by // a service must share the same value. @@ -387,7 +387,6 @@ func (p *payloadV1) encodeSpans(bm bitmap, fieldID int, spans spanList, st *stri if span == nil { continue } - // In encodeSpans function, after line 388: p.buf = msgp.AppendMapHeader(p.buf, 16) // number of fields in span p.buf = encodeField(p.buf, fullSetBitmap, 1, span.service, st) @@ -434,7 +433,6 @@ func (p *payloadV1) encodeSpans(bm bitmap, fieldID int, spans spanList, st *stri component := span.meta[ext.Component] p.buf = encodeField(p.buf, fullSetBitmap, 15, component, st) - // And especially field 16: spanKind := span.meta[ext.SpanKind] p.buf = encodeField(p.buf, fullSetBitmap, 16, spanKind, st) } @@ -453,8 +451,6 @@ func (p *payloadV1) encodeSpanLinks(bm bitmap, fieldID int, spanLinks []SpanLink p.buf = encodeField(p.buf, fullSetBitmap, 1, link.TraceID, st) p.buf = encodeField(p.buf, fullSetBitmap, 2, link.SpanID, st) - p.buf = encodeField(p.buf, fullSetBitmap, 4, link.Tracestate, st) - p.buf = encodeField(p.buf, fullSetBitmap, 5, link.Flags, st) attr := map[string]anyValue{} for k, v := range link.Attributes { @@ -464,6 +460,9 @@ func (p *payloadV1) encodeSpanLinks(bm bitmap, fieldID int, spanLinks []SpanLink } } p.encodeAttributes(fullSetBitmap, 3, attr, st) + + p.buf = encodeField(p.buf, fullSetBitmap, 4, link.Tracestate, st) + p.buf = encodeField(p.buf, fullSetBitmap, 5, link.Flags, st) } return true, nil } @@ -625,7 +624,7 @@ func (p *payloadV1) decodeBuffer() ([]byte, error) { var ok bool value, o, ok = st.Read(o) if !ok { - err = fmt.Errorf("unable to read string value of field %d", idx) + err = errUnableDecodeString break } @@ -1111,6 +1110,9 @@ func (link *SpanLink) decode(b []byte, st *stringTable) ([]byte, error) { return b, err } for range numFields { + if err != nil { + return b, err + } // read msgp field ID var idx uint32 idx, o, err = msgp.ReadUint32Bytes(o) @@ -1122,20 +1124,11 @@ func (link *SpanLink) decode(b []byte, st *stringTable) ([]byte, error) { switch idx { case 1: link.TraceID, o, err = msgp.ReadUint64Bytes(o) - if err != nil { - return o, err - } case 2: link.SpanID, o, err = msgp.ReadUint64Bytes(o) - if err != nil { - return o, err - } case 3: var attr map[string]anyValue attr, o, err = DecodeKeyValueList(o, st) - if err != nil { - return o, err - } for k, v := range attr { if v.valueType != StringValueType { return o, fmt.Errorf("unexpected value type: %d", v.valueType) @@ -1143,15 +1136,16 @@ func (link *SpanLink) decode(b []byte, st *stringTable) ([]byte, error) { link.Attributes[k] = v.value.(string) } case 4: - link.Tracestate, o, err = msgp.ReadStringBytes(o) - if err != nil { - return o, err + var state string + var ok bool + state, o, ok = st.Read(o) + if !ok { + err = errUnableDecodeString + break } + link.Tracestate = state case 5: link.Flags, o, err = msgp.ReadUint32Bytes(o) - if err != nil { - return o, err - } default: return o, fmt.Errorf("unexpected field ID %d", idx) } @@ -1195,10 +1189,14 @@ func (event *spanEvent) decode(b []byte, st *stringTable) ([]byte, error) { return o, err } case 2: - event.Name, o, err = msgp.ReadStringBytes(o) - if err != nil { - return o, err + var name string + var ok bool + name, o, ok = st.Read(o) + if !ok { + err = errUnableDecodeString + break } + event.Name = name case 3: var attr map[string]anyValue attr, o, err = DecodeKeyValueList(o, st) From 91c5929cb6391f27e96f6a7fc42e055ea2adc171 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Fri, 10 Oct 2025 12:59:40 -0400 Subject: [PATCH 42/75] add span links and events to detailed test --- ddtrace/tracer/payload_test.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ddtrace/tracer/payload_test.go b/ddtrace/tracer/payload_test.go index 31c089276f..c0644625a5 100644 --- a/ddtrace/tracer/payload_test.go +++ b/ddtrace/tracer/payload_test.go @@ -46,8 +46,8 @@ func newDetailedSpanList(n int) spanList { list[i].resource = "resource." + itoa[i%5+1] list[i].error = int32(i % 2) list[i].SetTag("tag."+itoa[i%5+1], "value."+itoa[i%5+1]) - // list[i].spanLinks = []SpanLink{{TraceID: 1, SpanID: 1}, {TraceID: 2, SpanID: 2}} - // list[i].spanEvents = []spanEvent{{Name: "span.event." + itoa[i%5+1]}} + list[i].spanLinks = []SpanLink{{TraceID: 1, SpanID: 1}, {TraceID: 2, SpanID: 2}} + list[i].spanEvents = []spanEvent{{Name: "span.event." + itoa[i%5+1]}} } return list } @@ -166,8 +166,9 @@ func TestPayloadV1Decode(t *testing.T) { assert.NotEmpty(got.attributes) assert.Equal(p.attributes, got.attributes) assert.Equal(got.attributes["service"].value, "golden") - assert.Equal(p.chunks[0].traceID, got.chunks[0].traceID) - assert.Equal(p.chunks[0].spans[0].spanID, got.chunks[0].spans[0].spanID) + assert.Greater(len(got.chunks), 0) + // assert.Equal(p.chunks[0].traceID, got.chunks[0].traceID) + // assert.Equal(p.chunks[0].spans[0].spanID, got.chunks[0].spans[0].spanID) }) } } From 82c805b60640cdc421ee41deb3f595531a58d2bd Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Fri, 10 Oct 2025 15:41:08 -0400 Subject: [PATCH 43/75] document functions and update api.txt --- ddtrace/tracer/api.txt | 9 +++++++++ ddtrace/tracer/payload_test.go | 4 ++-- ddtrace/tracer/payload_v1.go | 28 +++++++++++++++++++++++++--- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/ddtrace/tracer/api.txt b/ddtrace/tracer/api.txt index caa9faa6ba..76be7db4e7 100644 --- a/ddtrace/tracer/api.txt +++ b/ddtrace/tracer/api.txt @@ -126,6 +126,15 @@ type UserMonitoringConfig struct { type UserMonitoringOption func(*UserMonitoringConfig)() +// File: payload_v1.go + +// Package Functions +func DecodeKeyValueList([]byte, *stringTable) (map[string]anyValue, []byte, error) +func DecodeSpanEvents([]byte, *stringTable) ([]spanEvent, []byte, error) +func DecodeSpanLinks([]byte, *stringTable) ([]SpanLink, []byte, error) +func DecodeSpans([]byte, *stringTable) (spanList, []byte, error) +func DecodeTraceChunks([]byte, *stringTable) ([]traceChunk, []byte, error) + // File: propagator.go // Types diff --git a/ddtrace/tracer/payload_test.go b/ddtrace/tracer/payload_test.go index c0644625a5..1fbc457d97 100644 --- a/ddtrace/tracer/payload_test.go +++ b/ddtrace/tracer/payload_test.go @@ -167,8 +167,8 @@ func TestPayloadV1Decode(t *testing.T) { assert.Equal(p.attributes, got.attributes) assert.Equal(got.attributes["service"].value, "golden") assert.Greater(len(got.chunks), 0) - // assert.Equal(p.chunks[0].traceID, got.chunks[0].traceID) - // assert.Equal(p.chunks[0].spans[0].spanID, got.chunks[0].spans[0].spanID) + assert.Equal(p.chunks[0].traceID, got.chunks[0].traceID) + assert.Equal(p.chunks[0].spans[0].spanID, got.chunks[0].spans[0].spanID) }) } } diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 892c5e68d0..c35a5d0079 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -187,6 +187,7 @@ func (p *payloadV1) clear() { p.reader = nil } +// recordItem records that a new chunk was added to the payload. func (p *payloadV1) recordItem() { atomic.AddUint32(&p.count, 1) } @@ -317,6 +318,7 @@ func encodeField[F fieldValue](buf []byte, bm bitmap, fieldID uint32, a F, st *s return buf } +// encodeAttributes encodes a keyValueList (most often attributes) associated with fieldID into the buffer in msgp format. func (p *payloadV1) encodeAttributes(bm bitmap, fieldID int, kv map[string]anyValue, st *stringTable) (bool, error) { if !bm.contains(uint32(fieldID)) { return false, nil @@ -340,6 +342,7 @@ func (p *payloadV1) encodeAttributes(bm bitmap, fieldID int, kv map[string]anyVa return true, nil } +// encodeTraceChunks encodes a list of trace chunks associated with fieldID into p.buf in msgp format. func (p *payloadV1) encodeTraceChunks(bm bitmap, fieldID int, tc []traceChunk, st *stringTable) (bool, error) { if len(tc) == 0 || !bm.contains(uint32(fieldID)) { return false, nil @@ -375,6 +378,7 @@ func (p *payloadV1) encodeTraceChunks(bm bitmap, fieldID int, tc []traceChunk, s return true, nil } +// encodeSpans encodes a list of spans associated with fieldID into p.buf in msgp format. func (p *payloadV1) encodeSpans(bm bitmap, fieldID int, spans spanList, st *stringTable) (bool, error) { if len(spans) == 0 || !bm.contains(uint32(fieldID)) { return false, nil @@ -439,6 +443,7 @@ func (p *payloadV1) encodeSpans(bm bitmap, fieldID int, spans spanList, st *stri return true, nil } +// encodeSpanLinks encodes a list of span links associated with fieldID into p.buf in msgp format. func (p *payloadV1) encodeSpanLinks(bm bitmap, fieldID int, spanLinks []SpanLink, st *stringTable) (bool, error) { if !bm.contains(uint32(fieldID)) { return false, nil @@ -467,6 +472,7 @@ func (p *payloadV1) encodeSpanLinks(bm bitmap, fieldID int, spanLinks []SpanLink return true, nil } +// encodeSpanEvents encodes a list of span events associated with fieldID into p.buf in msgp format. func (p *payloadV1) encodeSpanEvents(bm bitmap, fieldID int, spanEvents []spanEvent, st *stringTable) (bool, error) { if !bm.contains(uint32(fieldID)) { return false, nil @@ -677,6 +683,7 @@ const ( keyValueListType // []keyValue -- 7 ) +// buildAnyValue builds an anyValue from a given any type. func buildAnyValue(v any) *anyValue { switch v := v.(type) { case string: @@ -743,6 +750,7 @@ type arrayValue []anyValue // 1 for represented fields and 0 for unset fields. type bitmap int32 +// fullSetBitmap is a bitmap that represents all fields that have been set in the payload. var fullSetBitmap bitmap = -1 func (b *bitmap) set(bit uint32) { @@ -791,6 +799,8 @@ func (s *stringValue) decode(buf []byte) ([]byte, error) { return o, nil } +var errUnableDecodeString = errors.New("unable to read string value") + type stringTable struct { strings []stringValue // list of strings indices map[stringValue]index // map strings to their indices @@ -805,6 +815,7 @@ func newStringTable() *stringTable { } } +// Adds a string to the string table if it does not already exist. Returns the index of the string. func (s *stringTable) Add(str string) (idx index) { sv := stringValue(str) if _, ok := s.indices[sv]; ok { @@ -817,6 +828,7 @@ func (s *stringTable) Add(str string) (idx index) { return } +// Get returns the index of a string in the string table if it exists. Returns false if the string does not exist. func (s *stringTable) Get(str string) (index, bool) { sv := stringValue(str) if idx, ok := s.indices[sv]; ok { @@ -825,6 +837,8 @@ func (s *stringTable) Get(str string) (index, bool) { return -1, false } +// Reads a string from a byte slice and returns it from the string table if it exists. +// Returns false if the string does not exist. func (s *stringTable) Read(b []byte) (string, []byte, bool) { sType := getStreamingType(b[0]) if sType == -1 { @@ -898,7 +912,7 @@ type traceChunk struct { samplingMechanism uint32 } -// Decoding Functions +// DecodeTraceChunks decodes a list of trace chunks from a byte slice. func DecodeTraceChunks(b []byte, st *stringTable) ([]traceChunk, []byte, error) { out := []traceChunk{} numChunks, o, err := msgp.ReadArrayHeaderBytes(b) @@ -959,6 +973,7 @@ func (tc *traceChunk) decode(b []byte, st *stringTable) ([]byte, error) { return o, err } +// DecodeSpans decodes a list of spans from a byte slice. func DecodeSpans(b []byte, st *stringTable) (spanList, []byte, error) { out := spanList{} numSpans, o, err := msgp.ReadArrayHeaderBytes(b) @@ -976,6 +991,8 @@ func DecodeSpans(b []byte, st *stringTable) (spanList, []byte, error) { return out, o, nil } +// decode reads a span from a byte slice and populates the associated fields in the span. +// This should only be used with decoding v1.0 payloads. func (span *Span) decode(b []byte, st *stringTable) ([]byte, error) { numFields, o, err := msgp.ReadMapHeaderBytes(b) for range numFields { @@ -1087,6 +1104,7 @@ func (span *Span) decode(b []byte, st *stringTable) ([]byte, error) { return o, err } +// DecodeSpanLinks decodes a list of span links from a byte slice. func DecodeSpanLinks(b []byte, st *stringTable) ([]SpanLink, []byte, error) { out := []SpanLink{} numLinks, o, err := msgp.ReadArrayHeaderBytes(b) @@ -1104,6 +1122,8 @@ func DecodeSpanLinks(b []byte, st *stringTable) ([]SpanLink, []byte, error) { return out, o, nil } +// decode reads a span link from a byte slice and populates the associated fields in the span link. +// This should only be used with decoding v1.0 payloads. func (link *SpanLink) decode(b []byte, st *stringTable) ([]byte, error) { numFields, o, err := msgp.ReadMapHeaderBytes(b) if err != nil { @@ -1153,6 +1173,7 @@ func (link *SpanLink) decode(b []byte, st *stringTable) ([]byte, error) { return o, err } +// DecodeSpanEvents decodes a list of span events from a byte slice. func DecodeSpanEvents(b []byte, st *stringTable) ([]spanEvent, []byte, error) { out := []spanEvent{} numEvents, o, err := msgp.ReadArrayHeaderBytes(b) @@ -1170,6 +1191,8 @@ func DecodeSpanEvents(b []byte, st *stringTable) ([]spanEvent, []byte, error) { return out, o, nil } +// decode reads a span event from a byte slice and populates the associated fields in the span event. +// This should only be used with decoding v1.0 payloads. func (event *spanEvent) decode(b []byte, st *stringTable) ([]byte, error) { numFields, o, err := msgp.ReadMapHeaderBytes(b) if err != nil { @@ -1215,8 +1238,6 @@ func (event *spanEvent) decode(b []byte, st *stringTable) ([]byte, error) { return o, err } -var errUnableDecodeString = errors.New("unable to read string value") - func decodeAnyValue(b []byte, strings *stringTable) (anyValue, []byte, error) { vType, o, err := msgp.ReadInt32Bytes(b) if err != nil { @@ -1288,6 +1309,7 @@ func decodeAnyValue(b []byte, strings *stringTable) (anyValue, []byte, error) { } } +// DecodeKeyValueList decodes a map of string to anyValue from a byte slice. func DecodeKeyValueList(b []byte, strings *stringTable) (map[string]anyValue, []byte, error) { numFields, o, err := msgp.ReadMapHeaderBytes(b) if err != nil { From bbb62d23f06c1067b24cfdda985a34d874930681 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dario=20Casta=C3=B1=C3=A9?= Date: Thu, 16 Oct 2025 11:26:05 +0200 Subject: [PATCH 44/75] chore: apply changes from #4010 --- ddtrace/tracer/payload_v04.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/ddtrace/tracer/payload_v04.go b/ddtrace/tracer/payload_v04.go index 54c99b1800..8233951a19 100644 --- a/ddtrace/tracer/payload_v04.go +++ b/ddtrace/tracer/payload_v04.go @@ -11,6 +11,7 @@ import ( "io" "sync/atomic" + "github.com/DataDog/dd-trace-go/v2/internal/processtags" "github.com/tinylib/msgp/msgp" ) @@ -76,6 +77,7 @@ func newPayloadV04() *payloadV04 { // push pushes a new item into the stream. func (p *payloadV04) push(t spanList) (stats payloadStats, err error) { + p.setTracerTags(t) p.buf.Grow(t.Msgsize()) if err := msgp.Encode(&p.buf, t); err != nil { return payloadStats{}, err @@ -84,6 +86,21 @@ func (p *payloadV04) push(t spanList) (stats payloadStats, err error) { return p.stats(), nil } +func (p *payloadV04) setTracerTags(t spanList) { + // set on first chunk + if atomic.LoadUint32(&p.count) != 0 { + return + } + if len(t) == 0 { + return + } + pTags := processtags.GlobalTags().String() + if pTags == "" { + return + } + t[0].setProcessTags(pTags) +} + // itemCount returns the number of items available in the stream. func (p *payloadV04) itemCount() int { return int(atomic.LoadUint32(&p.count)) From 72fae1dd43b82c96e6aee9ed2c55249dae5b75ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dario=20Casta=C3=B1=C3=A9?= Date: Thu, 16 Oct 2025 12:09:32 +0200 Subject: [PATCH 45/75] chore: go fmt --- ddtrace/tracer/payload_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/tracer/payload_test.go b/ddtrace/tracer/payload_test.go index 6891157c8a..7a5895803d 100644 --- a/ddtrace/tracer/payload_test.go +++ b/ddtrace/tracer/payload_test.go @@ -234,7 +234,7 @@ func assertProcessTags(t *testing.T, payload spanLists) { } require.False(t, ok, "process tags should be present on the first span of each chunk only (chunk: %d span: %d)", i, j) } - } + } } func BenchmarkPayloadThroughput(b *testing.B) { From 7d4c9cfac6d7ff4548bd1a863042ee6a73c20206 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dario=20Casta=C3=B1=C3=A9?= Date: Thu, 16 Oct 2025 17:20:03 +0200 Subject: [PATCH 46/75] feat(.github/workflows): add APM_TRACING_EFFICIENT_PAYLOAD scenario for system-tests --- .github/workflows/system-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index f38b40fdf5..e220464c58 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -136,6 +136,8 @@ jobs: scenario: AGENT_SUPPORTING_SPAN_EVENTS - weblog-variant: net-http scenario: TELEMETRY_METRIC_GENERATION_DISABLED + - weblog-variant: net-http + scenario: APM_TRACING_EFFICIENT_PAYLOAD fail-fast: false env: TEST_LIBRARY: golang From a989bde6ff22e79e43f3979c133a9fb8aa9b9fd2 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Mon, 20 Oct 2025 15:04:00 -0400 Subject: [PATCH 47/75] replace DD_TRACE_AGENT_PROTOCOL_VERSION with DD_TRACE_V1_PAYLOAD_FORMAT_ENABLED --- ddtrace/tracer/option.go | 12 +++++++++--- ddtrace/tracer/transport.go | 5 +++-- ddtrace/tracer/writer_test.go | 6 +++--- internal/env/supported_configurations.gen.go | 2 +- internal/env/supported_configurations.json | 6 +++--- 5 files changed, 19 insertions(+), 12 deletions(-) diff --git a/ddtrace/tracer/option.go b/ddtrace/tracer/option.go index 2413a6a624..f5c90a0f02 100644 --- a/ddtrace/tracer/option.go +++ b/ddtrace/tracer/option.go @@ -410,9 +410,6 @@ func newConfig(opts ...StartOption) (*config, error) { reportTelemetryOnAppStarted(telemetry.Configuration{Name: "trace_rate_limit", Value: c.traceRateLimitPerSecond, Origin: origin}) - // Set the trace protocol to use. - c.traceProtocol = internal.FloatEnv("DD_TRACE_AGENT_PROTOCOL_VERSION", traceProtocolV04) - if v := env.Get("OTEL_LOGS_EXPORTER"); v != "" { log.Warn("OTEL_LOGS_EXPORTER is not supported") } @@ -588,6 +585,15 @@ func newConfig(opts ...StartOption) (*config, error) { if c.transport == nil { c.transport = newHTTPTransport(c.agentURL.String(), c.httpClient) } + // Set the trace protocol to use. + if internal.BoolEnv("DD_TRACE_V1_PAYLOAD_FORMAT_ENABLED", false) { + c.traceProtocol = traceProtocolV1 + if t, ok := c.transport.(*httpTransport); ok { + t.traceURL = tracesAPIPathV1 + } + } else { + c.traceProtocol = traceProtocolV04 + } if c.propagator == nil { envKey := "DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH" maxLen := internal.IntEnv(envKey, defaultMaxTagsHeaderLen) diff --git a/ddtrace/tracer/transport.go b/ddtrace/tracer/transport.go index aa6c1981d6..17e92f4b66 100644 --- a/ddtrace/tracer/transport.go +++ b/ddtrace/tracer/transport.go @@ -66,8 +66,9 @@ const ( traceCountHeader = "X-Datadog-Trace-Count" // header containing the number of traces in the payload obfuscationVersionHeader = "Datadog-Obfuscation-Version" // header containing the version of obfuscation used, if any - tracesAPIPath = "/v0.4/traces" - statsAPIPath = "/v0.6/stats" + tracesAPIPath = "/v0.4/traces" + tracesAPIPathV1 = "/v1.0/traces" + statsAPIPath = "/v0.6/stats" ) // transport is an interface for communicating data to the agent. diff --git a/ddtrace/tracer/writer_test.go b/ddtrace/tracer/writer_test.go index 74a55558b7..a873fb14d1 100644 --- a/ddtrace/tracer/writer_test.go +++ b/ddtrace/tracer/writer_test.go @@ -455,7 +455,7 @@ func TestTraceProtocol(t *testing.T) { assert := assert.New(t) t.Run("v1.0", func(t *testing.T) { - t.Setenv("DD_TRACE_AGENT_PROTOCOL_VERSION", "1.0") + t.Setenv("DD_TRACE_V1_PAYLOAD_FORMAT_ENABLED", "true") cfg, err := newTestConfig() require.NoError(t, err) h := newAgentTraceWriter(cfg, nil, nil) @@ -463,7 +463,7 @@ func TestTraceProtocol(t *testing.T) { }) t.Run("v0.4", func(t *testing.T) { - t.Setenv("DD_TRACE_AGENT_PROTOCOL_VERSION", "0.4") + t.Setenv("DD_TRACE_V1_PAYLOAD_FORMAT_ENABLED", "false") cfg, err := newTestConfig() require.NoError(t, err) h := newAgentTraceWriter(cfg, nil, nil) @@ -478,7 +478,7 @@ func TestTraceProtocol(t *testing.T) { }) t.Run("invalid", func(t *testing.T) { - t.Setenv("DD_TRACE_AGENT_PROTOCOL_VERSION", "invalid") + t.Setenv("DD_TRACE_V1_PAYLOAD_FORMAT_ENABLED", "invalid") cfg, err := newTestConfig() require.NoError(t, err) h := newAgentTraceWriter(cfg, nil, nil) diff --git a/internal/env/supported_configurations.gen.go b/internal/env/supported_configurations.gen.go index 2ae4274dd2..8a219798c6 100644 --- a/internal/env/supported_configurations.gen.go +++ b/internal/env/supported_configurations.gen.go @@ -128,7 +128,7 @@ var SupportedConfigurations = map[string]struct{}{ "DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED": {}, "DD_TRACE_ABANDONED_SPAN_TIMEOUT": {}, "DD_TRACE_AGENT_PORT": {}, - "DD_TRACE_AGENT_PROTOCOL_VERSION": {}, + "DD_TRACE_V1_PAYLOAD_FORMAT_ENABLED": {}, "DD_TRACE_AGENT_URL": {}, "DD_TRACE_ANALYTICS_ENABLED": {}, "DD_TRACE_AWS_ANALYTICS_ENABLED": {}, diff --git a/internal/env/supported_configurations.json b/internal/env/supported_configurations.json index e8a2e86411..f5dca931ab 100644 --- a/internal/env/supported_configurations.json +++ b/internal/env/supported_configurations.json @@ -357,9 +357,6 @@ "DD_TRACE_AGENT_PORT": [ "A" ], - "DD_TRACE_AGENT_PROTOCOL_VERSION": [ - "A" - ], "DD_TRACE_AGENT_URL": [ "A" ], @@ -594,6 +591,9 @@ "DD_TRACE_TWIRP_ANALYTICS_ENABLED": [ "A" ], + "DD_TRACE_V1_PAYLOAD_FORMAT_ENABLED": [ + "A" + ], "DD_TRACE_VALKEY_ANALYTICS_ENABLED": [ "A" ], From e2ef0d588e7b640d00b86a37233ec126c9fcb51c Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Mon, 20 Oct 2025 15:10:55 -0400 Subject: [PATCH 48/75] initialize header with 8 bytes can this be done more efficiently? --- ddtrace/tracer/payload_v1.go | 1 + 1 file changed, 1 insertion(+) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index c35a5d0079..1e95768fc9 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -97,6 +97,7 @@ type payloadV1 struct { func newPayloadV1() *payloadV1 { return &payloadV1{ protocolVersion: traceProtocolV1, + header: make([]byte, 8), attributes: make(map[string]anyValue), chunks: make([]traceChunk, 0), readOff: 8, From 00670181e5991d8c18d27e3177f278542290ee3d Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Mon, 20 Oct 2025 15:27:17 -0400 Subject: [PATCH 49/75] fix: move init of header size to inside updateHeader --- ddtrace/tracer/payload_v1.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 1e95768fc9..4ee9713536 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -97,7 +97,6 @@ type payloadV1 struct { func newPayloadV1() *payloadV1 { return &payloadV1{ protocolVersion: traceProtocolV1, - header: make([]byte, 8), attributes: make(map[string]anyValue), chunks: make([]traceChunk, 0), readOff: 8, @@ -213,6 +212,9 @@ func (p *payloadV1) protocol() float64 { } func (p *payloadV1) updateHeader() { + if len(p.header) == 0 { + p.header = make([]byte, 8) + } n := atomic.LoadUint32(&p.fields) switch { case n <= 15: From 5e1633d8135a0c764f71e42d209cda4ce6dfcb0a Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Mon, 20 Oct 2025 16:12:07 -0400 Subject: [PATCH 50/75] fix: use proper traceURL for v1 protocol --- ddtrace/tracer/option.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/tracer/option.go b/ddtrace/tracer/option.go index f5c90a0f02..80445a4436 100644 --- a/ddtrace/tracer/option.go +++ b/ddtrace/tracer/option.go @@ -589,7 +589,7 @@ func newConfig(opts ...StartOption) (*config, error) { if internal.BoolEnv("DD_TRACE_V1_PAYLOAD_FORMAT_ENABLED", false) { c.traceProtocol = traceProtocolV1 if t, ok := c.transport.(*httpTransport); ok { - t.traceURL = tracesAPIPathV1 + t.traceURL = fmt.Sprintf("%s%s", c.agentURL.String(), tracesAPIPathV1) } } else { c.traceProtocol = traceProtocolV04 From 888c82c1def40435c70a8764b7440b30fdefb698 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Tue, 21 Oct 2025 15:43:12 -0400 Subject: [PATCH 51/75] fix: encode attributes as a list instead of map --- ddtrace/tracer/payload_v1.go | 79 ++++++++++++++++++++++++------------ 1 file changed, 52 insertions(+), 27 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 4ee9713536..4e6b4e4797 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -293,13 +293,7 @@ func encodeField[F fieldValue](buf []byte, bm bitmap, fieldID uint32, a F, st *s switch value := any(a).(type) { case string: // encode msgp value, either by pulling from string table or writing it directly - if idx, ok := st.Get(value); ok { - buf = idx.encode(buf) - } else { - s := stringValue(value) - buf = s.encode(buf) - st.Add(value) - } + buf = st.Serialize(value, buf) case bool: buf = msgp.AppendBool(buf, value) case float64: @@ -321,23 +315,18 @@ func encodeField[F fieldValue](buf []byte, bm bitmap, fieldID uint32, a F, st *s return buf } -// encodeAttributes encodes a keyValueList (most often attributes) associated with fieldID into the buffer in msgp format. +// encodeAttributes encodes an array associated with fieldID into the buffer in msgp format. func (p *payloadV1) encodeAttributes(bm bitmap, fieldID int, kv map[string]anyValue, st *stringTable) (bool, error) { if !bm.contains(uint32(fieldID)) { return false, nil } - p.buf = msgp.AppendUint32(p.buf, uint32(fieldID)) // msgp key - p.buf = msgp.AppendMapHeader(p.buf, uint32(len(kv))) // number of item pairs in map + p.buf = msgp.AppendUint32(p.buf, uint32(fieldID)) // msgp key + p.buf = msgp.AppendArrayHeader(p.buf, uint32(len(kv)*3)) // number of item pairs in map for k, v := range kv { // encode msgp key - if idx, ok := st.Get(string(k)); ok { - p.buf = idx.encode(p.buf) - } else { - p.buf = stringValue(k).encode(p.buf) - st.Add(string(k)) - } + p.buf = st.Serialize(k, p.buf) // encode value p.buf = v.encode(p.buf, st) @@ -612,7 +601,7 @@ func (p *payloadV1) decodeBuffer() ([]byte, error) { // handle attributes if idx == 10 { - p.attributes, o, err = DecodeKeyValueList(o, st) + p.attributes, o, err = DecodeAttributes(o, st) if err != nil { break } @@ -711,12 +700,7 @@ func (a anyValue) encode(buf []byte, st *stringTable) []byte { switch a.valueType { case StringValueType: s := a.value.(string) - if idx, ok := st.Get(s); ok { - buf = idx.encode(buf) - } else { - buf = stringValue(s).encode(buf) - st.Add(s) - } + buf = st.Serialize(s, buf) case BoolValueType: buf = msgp.AppendBool(buf, a.value.(bool)) case FloatValueType: @@ -790,6 +774,7 @@ func (i *index) decode(buf []byte) ([]byte, error) { type stringValue string func (s stringValue) encode(buf []byte) []byte { + // TODO(hannahkm): add the fixstr representation return msgp.AppendString(buf, string(s)) } @@ -840,6 +825,17 @@ func (s *stringTable) Get(str string) (index, bool) { return -1, false } +func (st *stringTable) Serialize(value string, buf []byte) []byte { + if idx, ok := st.Get(value); ok { + buf = idx.encode(buf) + } else { + s := stringValue(value) + buf = s.encode(buf) + st.Add(value) + } + return buf +} + // Reads a string from a byte slice and returns it from the string table if it exists. // Returns false if the string does not exist. func (s *stringTable) Read(b []byte) (string, []byte, bool) { @@ -960,7 +956,7 @@ func (tc *traceChunk) decode(b []byte, st *stringTable) ([]byte, error) { err = errUnableDecodeString } case 3: - tc.attributes, o, err = DecodeKeyValueList(o, st) + tc.attributes, o, err = DecodeAttributes(o, st) case 4: tc.spans, o, err = DecodeSpans(o, st) case 5: @@ -1047,7 +1043,7 @@ func (span *Span) decode(b []byte, st *stringTable) ([]byte, error) { } case 9: var attr map[string]anyValue - attr, o, err = DecodeKeyValueList(o, st) + attr, o, err = DecodeAttributes(o, st) for k, v := range attr { span.SetTag(k, v.value) } @@ -1151,7 +1147,7 @@ func (link *SpanLink) decode(b []byte, st *stringTable) ([]byte, error) { link.SpanID, o, err = msgp.ReadUint64Bytes(o) case 3: var attr map[string]anyValue - attr, o, err = DecodeKeyValueList(o, st) + attr, o, err = DecodeAttributes(o, st) for k, v := range attr { if v.valueType != StringValueType { return o, fmt.Errorf("unexpected value type: %d", v.valueType) @@ -1225,7 +1221,7 @@ func (event *spanEvent) decode(b []byte, st *stringTable) ([]byte, error) { event.Name = name case 3: var attr map[string]anyValue - attr, o, err = DecodeKeyValueList(o, st) + attr, o, err = DecodeAttributes(o, st) if err != nil { break } @@ -1338,3 +1334,32 @@ func DecodeKeyValueList(b []byte, strings *stringTable) (map[string]anyValue, [] } return kv, o, nil } + +// DecodeAttributes decodes a map of string to anyValue from a byte slice +// Attributes are encoded as an array of key, valueType, and value. +func DecodeAttributes(b []byte, strings *stringTable) (map[string]anyValue, []byte, error) { + n, o, err := msgp.ReadArrayHeaderBytes(b) + numFields := n / 3 + if err != nil { + return nil, b, err + } + + kv := map[string]anyValue{} + for i := range numFields { + var ( + key string + ok bool + av anyValue + ) + key, o, ok = strings.Read(o) + if !ok { + return nil, o, fmt.Errorf("unable to read key of field %d", i) + } + av, o, err = decodeAnyValue(o, strings) + if err != nil { + return nil, o, err + } + kv[key] = av + } + return kv, o, nil +} From c4b126d647f0480bc20cfdb7e72e4d86d45cf361 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Tue, 21 Oct 2025 15:52:24 -0400 Subject: [PATCH 52/75] update api txt file --- ddtrace/tracer/api.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/ddtrace/tracer/api.txt b/ddtrace/tracer/api.txt index 83180d7a3c..36893e6f5c 100644 --- a/ddtrace/tracer/api.txt +++ b/ddtrace/tracer/api.txt @@ -133,6 +133,7 @@ type UserMonitoringOption func(*UserMonitoringConfig)() // File: payload_v1.go // Package Functions +func DecodeAttributes([]byte, *stringTable) (map[string]anyValue, []byte, error) func DecodeKeyValueList([]byte, *stringTable) (map[string]anyValue, []byte, error) func DecodeSpanEvents([]byte, *stringTable) ([]spanEvent, []byte, error) func DecodeSpanLinks([]byte, *stringTable) ([]SpanLink, []byte, error) From d931d7ec3f4dd3a472c46b194f6d914b7e74d352 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Tue, 21 Oct 2025 16:37:24 -0400 Subject: [PATCH 53/75] fix: properly set traceID --- ddtrace/tracer/payload_v1.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 4e6b4e4797..4dcd09da02 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -115,7 +115,7 @@ func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { // For now, we blindly set the origin, priority, and attributes values for the chunk // In the future, attributes should hold values that are shared across all chunks in the payload - origin, priority, sm := "", 0, 0 + origin, priority, sm, traceID := "", 0, 0, []byte{} for _, span := range t { if span == nil { continue @@ -125,6 +125,11 @@ func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { if _, ok := p.attributes["service"]; !ok { p.attributes["service"] = anyValue{valueType: StringValueType, value: span.Root().service} } + if len(traceID) == 0 { + b := make([]byte, 8) + binary.LittleEndian.PutUint64(b, span.traceID) + traceID = b + } if p, ok := span.Context().SamplingPriority(); ok { origin = span.Context().origin // TODO(darccio): are we sure that origin will be shared across all the spans in the chunk? priority = p // TODO(darccio): the same goes for priority. @@ -141,7 +146,7 @@ func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { spans: t, priority: int32(priority), origin: origin, - traceID: t[0].Context().traceID[:], + traceID: traceID, samplingMechanism: uint32(sm), } From a8b15f285897d3541ee9adfd698dabcaf89d642e Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Tue, 21 Oct 2025 17:09:01 -0400 Subject: [PATCH 54/75] fix: use big endian for trace id? --- ddtrace/tracer/payload_v1.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 4dcd09da02..46c16c9b24 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -127,7 +127,7 @@ func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { } if len(traceID) == 0 { b := make([]byte, 8) - binary.LittleEndian.PutUint64(b, span.traceID) + binary.BigEndian.PutUint64(b, span.traceID) traceID = b } if p, ok := span.Context().SamplingPriority(); ok { From 2ab205182b9d1823f546db6fc6c276b7cc499a87 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Wed, 22 Oct 2025 09:36:20 -0400 Subject: [PATCH 55/75] fix: encode both upper and lower traceID bits --- ddtrace/tracer/payload_v1.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 46c16c9b24..7926d11cea 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -115,7 +115,7 @@ func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { // For now, we blindly set the origin, priority, and attributes values for the chunk // In the future, attributes should hold values that are shared across all chunks in the payload - origin, priority, sm, traceID := "", 0, 0, []byte{} + origin, priority, sm, traceID := "", 0, 0, [16]byte{} for _, span := range t { if span == nil { continue @@ -126,9 +126,8 @@ func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { p.attributes["service"] = anyValue{valueType: StringValueType, value: span.Root().service} } if len(traceID) == 0 { - b := make([]byte, 8) - binary.BigEndian.PutUint64(b, span.traceID) - traceID = b + binary.BigEndian.PutUint64(traceID[:8], span.Context().traceID.Upper()) + binary.BigEndian.PutUint64(traceID[8:], span.Context().traceID.Lower()) } if p, ok := span.Context().SamplingPriority(); ok { origin = span.Context().origin // TODO(darccio): are we sure that origin will be shared across all the spans in the chunk? @@ -146,7 +145,7 @@ func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { spans: t, priority: int32(priority), origin: origin, - traceID: traceID, + traceID: traceID[:], samplingMechanism: uint32(sm), } From 092f6077b092b913b03964a8841e63f8d3f34fe9 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Wed, 22 Oct 2025 10:07:23 -0400 Subject: [PATCH 56/75] fix: traceID was never getting set --- ddtrace/tracer/payload_v1.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 7926d11cea..cce35bef77 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -125,10 +125,9 @@ func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { if _, ok := p.attributes["service"]; !ok { p.attributes["service"] = anyValue{valueType: StringValueType, value: span.Root().service} } - if len(traceID) == 0 { - binary.BigEndian.PutUint64(traceID[:8], span.Context().traceID.Upper()) - binary.BigEndian.PutUint64(traceID[8:], span.Context().traceID.Lower()) - } + binary.BigEndian.PutUint64(traceID[:8], span.Context().traceID.Upper()) + binary.BigEndian.PutUint64(traceID[8:], span.Context().traceID.Lower()) + if p, ok := span.Context().SamplingPriority(); ok { origin = span.Context().origin // TODO(darccio): are we sure that origin will be shared across all the spans in the chunk? priority = p // TODO(darccio): the same goes for priority. From 639f9b27f13f19b647b9382c1b913306340da684 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Wed, 22 Oct 2025 11:12:59 -0400 Subject: [PATCH 57/75] fix: empty payload should also be encoded as a map --- ddtrace/tracer/payload_test.go | 12 ++++++++++++ ddtrace/tracer/payload_v1.go | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/ddtrace/tracer/payload_test.go b/ddtrace/tracer/payload_test.go index 7a5895803d..f09e6f7260 100644 --- a/ddtrace/tracer/payload_test.go +++ b/ddtrace/tracer/payload_test.go @@ -200,6 +200,7 @@ func TestPayloadV1EmbeddedStreamingStringTable(t *testing.T) { func TestPayloadV1UpdateHeader(t *testing.T) { testCases := []uint32{ // Number of items + 0, 15, math.MaxUint16, math.MaxUint32, @@ -222,6 +223,17 @@ func TestPayloadV1UpdateHeader(t *testing.T) { } } +func TestEmptyPayloadV1(t *testing.T) { + p := newPayloadV1() + assert := assert.New(t) + encoded, err := io.ReadAll(p) + assert.NoError(err) + length, o, err := msgp.ReadMapHeaderBytes(encoded) + assert.NoError(err) + assert.Equal(uint32(0), length) + assert.Empty(o) +} + func assertProcessTags(t *testing.T, payload spanLists) { assert := assert.New(t) for i, spanList := range payload { diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index cce35bef77..29d4aac314 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -99,7 +99,7 @@ func newPayloadV1() *payloadV1 { protocolVersion: traceProtocolV1, attributes: make(map[string]anyValue), chunks: make([]traceChunk, 0), - readOff: 8, + readOff: 0, writeOff: 0, } } From 581c99438b511a7a61ace8845e8cc59ad8d82349 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Wed, 22 Oct 2025 13:03:25 -0400 Subject: [PATCH 58/75] fix: non atomic updates to fields, doc string fixes, etc --- ddtrace/tracer/payload_v1.go | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 29d4aac314..b6b3d6d43b 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -70,7 +70,7 @@ type payloadV1 struct { protocolVersion float64 // header specifies the first few bytes in the msgpack stream - // indicating the type of array (fixarray, array16 or array32) + // indicating the type of map (fixmap, map16 or map32) // and the number of items contained in the stream. header []byte @@ -110,7 +110,7 @@ func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { // Conceptually, our `t spanList` corresponds to one `traceChunk`. if !p.bm.contains(11) && len(t) > 0 { p.bm.set(11) - p.fields += 1 + atomic.AddUint32(&p.fields, 1) } // For now, we blindly set the origin, priority, and attributes values for the chunk @@ -152,7 +152,7 @@ func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { // the number of fields. if !p.bm.contains(10) && len(p.attributes) > 0 { p.bm.set(10) - p.fields += 1 + atomic.AddUint32(&p.fields, 1) } p.chunks = append(p.chunks, tc) @@ -186,8 +186,12 @@ func (p *payloadV1) reset() { func (p *payloadV1) clear() { p.bm = 0 - p.buf = p.buf[:] + p.buf = p.buf[:0] p.reader = nil + p.header = nil + p.readOff = 0 + p.fields = 0 + p.count = 0 } // recordItem records that a new chunk was added to the payload. @@ -319,13 +323,14 @@ func encodeField[F fieldValue](buf []byte, bm bitmap, fieldID uint32, a F, st *s } // encodeAttributes encodes an array associated with fieldID into the buffer in msgp format. +// Each attribute is encoded as three values: the key, value type, and value. func (p *payloadV1) encodeAttributes(bm bitmap, fieldID int, kv map[string]anyValue, st *stringTable) (bool, error) { if !bm.contains(uint32(fieldID)) { return false, nil } p.buf = msgp.AppendUint32(p.buf, uint32(fieldID)) // msgp key - p.buf = msgp.AppendArrayHeader(p.buf, uint32(len(kv)*3)) // number of item pairs in map + p.buf = msgp.AppendArrayHeader(p.buf, uint32(len(kv)*3)) // number of item pairs in array for k, v := range kv { // encode msgp key @@ -532,49 +537,49 @@ func (p *payloadV1) GetAttributes() map[string]anyValue { return p.attributes } func (p *payloadV1) SetContainerID(v string) { p.containerID = v p.bm.set(2) - p.fields += 1 + atomic.AddUint32(&p.fields, 1) } func (p *payloadV1) SetLanguageName(v string) { p.languageName = v p.bm.set(3) - p.fields += 1 + atomic.AddUint32(&p.fields, 1) } func (p *payloadV1) SetLanguageVersion(v string) { p.languageVersion = v p.bm.set(4) - p.fields += 1 + atomic.AddUint32(&p.fields, 1) } func (p *payloadV1) SetTracerVersion(v string) { p.tracerVersion = v p.bm.set(5) - p.fields += 1 + atomic.AddUint32(&p.fields, 1) } func (p *payloadV1) SetRuntimeID(v string) { p.runtimeID = v p.bm.set(6) - p.fields += 1 + atomic.AddUint32(&p.fields, 1) } func (p *payloadV1) SetEnv(v string) { p.env = v p.bm.set(7) - p.fields += 1 + atomic.AddUint32(&p.fields, 1) } func (p *payloadV1) SetHostname(v string) { p.hostname = v p.bm.set(8) - p.fields += 1 + atomic.AddUint32(&p.fields, 1) } func (p *payloadV1) SetAppVersion(v string) { p.appVersion = v p.bm.set(9) - p.fields += 1 + atomic.AddUint32(&p.fields, 1) } // decodeBuffer takes the buffer from the payload, decodes it, and populates the fields From 0487a5ac358bff8040e41dfb2f89b790117cd74f Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Wed, 22 Oct 2025 13:26:27 -0400 Subject: [PATCH 59/75] fix: initial protocol msg was sending an array --- ddtrace/tracer/log.go | 10 +++++++--- ddtrace/tracer/payload_v1.go | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/ddtrace/tracer/log.go b/ddtrace/tracer/log.go index d0ed37eef9..efe07b44ce 100644 --- a/ddtrace/tracer/log.go +++ b/ddtrace/tracer/log.go @@ -69,8 +69,12 @@ type startupInfo struct { // checkEndpoint tries to connect to the URL specified by endpoint. // If the endpoint is not reachable, checkEndpoint returns an error // explaining why. -func checkEndpoint(c *http.Client, endpoint string) error { - req, err := http.NewRequest("POST", endpoint, bytes.NewReader([]byte{0x90})) +func checkEndpoint(c *http.Client, endpoint string, protocol float64) error { + b := []byte{0x90} // empty array + if protocol == traceProtocolV1 { + b = []byte{0x80} // empty map + } + req, err := http.NewRequest("POST", endpoint, bytes.NewReader(b)) if err != nil { return fmt.Errorf("cannot create http request: %s", err) } @@ -162,7 +166,7 @@ func logStartup(t *tracer) { info.SampleRateLimit = fmt.Sprintf("%v", limit) } if !t.config.logToStdout { - if err := checkEndpoint(t.config.httpClient, t.config.transport.endpoint()); err != nil { + if err := checkEndpoint(t.config.httpClient, t.config.transport.endpoint(), t.config.traceProtocol); err != nil { info.AgentError = fmt.Sprintf("%s", err.Error()) log.Warn("DIAGNOSTICS Unable to reach agent intake: %s", err.Error()) } diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index b6b3d6d43b..be7a02d796 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -190,7 +190,7 @@ func (p *payloadV1) clear() { p.reader = nil p.header = nil p.readOff = 0 - p.fields = 0 + atomic.StoreUint32(&p.fields, 0) p.count = 0 } @@ -590,7 +590,7 @@ func (p *payloadV1) decodeBuffer() ([]byte, error) { return p.buf, err } p.buf = o - p.fields = numFields + atomic.StoreUint32(&p.fields, numFields) p.header = make([]byte, 8) p.updateHeader() From 8813fa571ab330cf815f72dfbbaec685e80027a8 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Wed, 22 Oct 2025 13:53:07 -0400 Subject: [PATCH 60/75] fix: sm was not properly represented as a uint32 --- ddtrace/tracer/payload_v1.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index be7a02d796..576b3e9484 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -132,14 +132,17 @@ func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { origin = span.Context().origin // TODO(darccio): are we sure that origin will be shared across all the spans in the chunk? priority = p // TODO(darccio): the same goes for priority. dm := span.context.trace.propagatingTag(keyDecisionMaker) - sm, err = strconv.Atoi(dm) // TODO(darccio): shouldn't this string represent the absolute value of dm? + sm, err = strconv.Atoi(dm) if err != nil { log.Error("failed to convert decision maker to int: %s", err.Error()) } break } } - + // we want to represent sampling mechanism as a uint32, the absolute value of the decision maker + if sm < 0 { + sm *= -1 + } tc := traceChunk{ spans: t, priority: int32(priority), From ff24c1f57004f56aa5b4514a07485b2f70d9ca51 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Wed, 22 Oct 2025 15:01:35 -0400 Subject: [PATCH 61/75] fix: change spankind to uint32 instead of a string --- ddtrace/tracer/payload_v1.go | 51 +++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 576b3e9484..1cd45ab59f 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -441,11 +441,47 @@ func (p *payloadV1) encodeSpans(bm bitmap, fieldID int, spans spanList, st *stri p.buf = encodeField(p.buf, fullSetBitmap, 15, component, st) spanKind := span.meta[ext.SpanKind] - p.buf = encodeField(p.buf, fullSetBitmap, 16, spanKind, st) + p.buf = encodeField(p.buf, fullSetBitmap, 16, getSpanKindValue(spanKind), st) } return true, nil } +// translate a span kind string to its uint32 value +func getSpanKindValue(sk string) uint32 { + switch sk { + case ext.SpanKindInternal: + return 1 + case ext.SpanKindServer: + return 2 + case ext.SpanKindClient: + return 3 + case ext.SpanKindProducer: + return 4 + case ext.SpanKindConsumer: + return 5 + default: + return 1 // default to internal + } +} + +// translate a span kind uint32 value to its string value +func getSpanKindString(sk uint32) string { + switch sk { + case 1: + return ext.SpanKindInternal + case 2: + return ext.SpanKindServer + case 3: + return ext.SpanKindClient + case 4: + return ext.SpanKindProducer + case 5: + return ext.SpanKindConsumer + default: + return ext.SpanKindInternal + } +} + // encodeSpanLinks encodes a list of span links associated with fieldID into p.buf in msgp format. func (p *payloadV1) encodeSpanLinks(bm bitmap, fieldID int, spanLinks []SpanLink, st *stringTable) (bool, error) { if !bm.contains(uint32(fieldID)) { @@ -1098,15 +1134,12 @@ func (span *Span) decode(b []byte, st *stringTable) ([]byte, error) { span.SetTag(ext.Component, component) } case 16: - var kind string - kind, o, ok = st.Read(o) - if !ok { - err = errUnableDecodeString - break - } - if kind != "" { - span.SetTag(ext.SpanKind, kind) + var sk uint32 + sk, o, err = msgp.ReadUint32Bytes(o) + if err != nil { + return o, err } + span.SetTag(ext.SpanKind, getSpanKindString(sk)) default: return o, fmt.Errorf("unexpected field ID %d", idx) } From 50724d819282bb0c73caa4fb4c04d0ffaa290198 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Thu, 23 Oct 2025 10:27:30 -0400 Subject: [PATCH 62/75] trigger tests From 883e3f23be2a77329f28a5c420bc4d6940cabf26 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Thu, 23 Oct 2025 14:53:58 -0400 Subject: [PATCH 63/75] feat: support process tags on spans --- ddtrace/tracer/payload_test.go | 8 ++++++++ ddtrace/tracer/payload_v1.go | 17 +++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/ddtrace/tracer/payload_test.go b/ddtrace/tracer/payload_test.go index f09e6f7260..9645d6fa24 100644 --- a/ddtrace/tracer/payload_test.go +++ b/ddtrace/tracer/payload_test.go @@ -171,6 +171,14 @@ func TestPayloadV1Decode(t *testing.T) { assert.Greater(len(got.chunks), 0) assert.Equal(p.chunks[0].traceID, got.chunks[0].traceID) assert.Equal(p.chunks[0].spans[0].spanID, got.chunks[0].spans[0].spanID) + + var hasTags bool + for _, chunk := range got.chunks { + for _, span := range chunk.spans { + hasTags = hasTags || assert.Contains(span.meta, keyProcessTags) + } + } + assert.True(hasTags) }) } } diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 1cd45ab59f..45c6d0fa32 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -15,6 +15,7 @@ import ( "github.com/DataDog/dd-trace-go/v2/ddtrace/ext" "github.com/DataDog/dd-trace-go/v2/internal/log" + "github.com/DataDog/dd-trace-go/v2/internal/processtags" "github.com/tinylib/msgp/msgp" ) @@ -112,6 +113,7 @@ func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { p.bm.set(11) atomic.AddUint32(&p.fields, 1) } + p.setTracerTags(t) // For now, we blindly set the origin, priority, and attributes values for the chunk // In the future, attributes should hold values that are shared across all chunks in the payload @@ -241,6 +243,21 @@ func (p *payloadV1) updateHeader() { } } +func (p *payloadV1) setTracerTags(t spanList) { + // set on first chunk + if atomic.LoadUint32(&p.count) != 0 { + return + } + if len(t) == 0 { + return + } + pTags := processtags.GlobalTags().String() + if pTags == "" { + return + } + t[0].setProcessTags(pTags) +} + func (p *payloadV1) Close() error { p.clear() return nil From bc024ceb212669b1e36a8cd506f29c73be9a4789 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Thu, 23 Oct 2025 14:58:48 -0400 Subject: [PATCH 64/75] chore: improve testing to specifically check for process tags on the very first span --- ddtrace/tracer/payload_test.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/ddtrace/tracer/payload_test.go b/ddtrace/tracer/payload_test.go index 9645d6fa24..ba229ac7a6 100644 --- a/ddtrace/tracer/payload_test.go +++ b/ddtrace/tracer/payload_test.go @@ -172,13 +172,8 @@ func TestPayloadV1Decode(t *testing.T) { assert.Equal(p.chunks[0].traceID, got.chunks[0].traceID) assert.Equal(p.chunks[0].spans[0].spanID, got.chunks[0].spans[0].spanID) - var hasTags bool - for _, chunk := range got.chunks { - for _, span := range chunk.spans { - hasTags = hasTags || assert.Contains(span.meta, keyProcessTags) - } - } - assert.True(hasTags) + // check that the first span of the first chunk has the process tags + assert.Contains(got.chunks[0].spans[0].meta, keyProcessTags) }) } } From e759d0eb2017365d4644034bfd8f0b8da36b6df0 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Thu, 23 Oct 2025 15:10:39 -0400 Subject: [PATCH 65/75] fix: set process tags on attributes instead of on chunks --- ddtrace/tracer/payload_test.go | 5 ++--- ddtrace/tracer/payload_v1.go | 14 +++++++------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/ddtrace/tracer/payload_test.go b/ddtrace/tracer/payload_test.go index ba229ac7a6..7c8fdfd430 100644 --- a/ddtrace/tracer/payload_test.go +++ b/ddtrace/tracer/payload_test.go @@ -17,6 +17,7 @@ import ( "testing" "github.com/DataDog/dd-trace-go/v2/internal/globalconfig" + "github.com/DataDog/dd-trace-go/v2/internal/processtags" "github.com/DataDog/dd-trace-go/v2/internal/version" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -168,12 +169,10 @@ func TestPayloadV1Decode(t *testing.T) { assert.NotEmpty(got.attributes) assert.Equal(p.attributes, got.attributes) assert.Equal(got.attributes["service"].value, "golden") + assert.Equal(got.attributes[keyProcessTags].value, processtags.GlobalTags().String()) assert.Greater(len(got.chunks), 0) assert.Equal(p.chunks[0].traceID, got.chunks[0].traceID) assert.Equal(p.chunks[0].spans[0].spanID, got.chunks[0].spans[0].spanID) - - // check that the first span of the first chunk has the process tags - assert.Contains(got.chunks[0].spans[0].meta, keyProcessTags) }) } } diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 45c6d0fa32..4c3ea68e3c 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -113,7 +113,6 @@ func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { p.bm.set(11) atomic.AddUint32(&p.fields, 1) } - p.setTracerTags(t) // For now, we blindly set the origin, priority, and attributes values for the chunk // In the future, attributes should hold values that are shared across all chunks in the payload @@ -158,6 +157,7 @@ func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { if !p.bm.contains(10) && len(p.attributes) > 0 { p.bm.set(10) atomic.AddUint32(&p.fields, 1) + p.setProcessTags() } p.chunks = append(p.chunks, tc) @@ -243,19 +243,19 @@ func (p *payloadV1) updateHeader() { } } -func (p *payloadV1) setTracerTags(t spanList) { - // set on first chunk +// Set process tags onto the payload attributes +func (p *payloadV1) setProcessTags() { if atomic.LoadUint32(&p.count) != 0 { return } - if len(t) == 0 { - return - } pTags := processtags.GlobalTags().String() if pTags == "" { return } - t[0].setProcessTags(pTags) + p.attributes[keyProcessTags] = anyValue{ + valueType: StringValueType, + value: pTags, + } } func (p *payloadV1) Close() error { From e4ae9134e4d76f4a0d9170f8a565ea4ace888618 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Fri, 24 Oct 2025 14:11:13 -0400 Subject: [PATCH 66/75] trigger tests From da2fd94fd16f2e49516162fd8cbb9c1833337ac6 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Mon, 27 Oct 2025 09:10:18 -0400 Subject: [PATCH 67/75] test against my system test branch --- .github/workflows/system-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index e220464c58..82a669a911 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -42,7 +42,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: 'DataDog/system-tests' - ref: ${{ inputs.ref }} + ref: hannahkm/implement-v1-serialization - name: Pin exact commit SHA for system-tests id: pin run: | From 813031f1dec65b8253bef057da8ffd95254198b0 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Mon, 27 Oct 2025 09:14:47 -0400 Subject: [PATCH 68/75] revert test --- .github/workflows/system-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index 82a669a911..e220464c58 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -42,7 +42,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: 'DataDog/system-tests' - ref: hannahkm/implement-v1-serialization + ref: ${{ inputs.ref }} - name: Pin exact commit SHA for system-tests id: pin run: | From 6ad1afeec385a356fd91fed7f75a4b54350b3eeb Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Mon, 27 Oct 2025 12:58:42 -0400 Subject: [PATCH 69/75] trigger tests From f2ecb715a21d53e6ebc512473834d6c1ef4f0185 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dario=20Casta=C3=B1=C3=A9?= Date: Wed, 29 Oct 2025 09:53:31 -0400 Subject: [PATCH 70/75] fix(internal/env): add new supported environment variable --- internal/env/supported_configurations.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/env/supported_configurations.json b/internal/env/supported_configurations.json index c83e6e852e..bb52dbdefd 100644 --- a/internal/env/supported_configurations.json +++ b/internal/env/supported_configurations.json @@ -612,6 +612,9 @@ "DD_TRACE_VAULT_ANALYTICS_ENABLED": [ "A" ], + "DD_TRACE_V1_PAYLOAD_FORMAT_ENABLED": [ + "A" + ], "DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH": [ "A" ], From ef59f664c599b38d18fb2280bce6b964fb0ec0db Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Wed, 29 Oct 2025 13:44:55 -0400 Subject: [PATCH 71/75] nit: remove unnecessary payload type field --- ddtrace/tracer/payload_v04.go | 10 +++------- ddtrace/tracer/payload_v1.go | 14 +++++--------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/ddtrace/tracer/payload_v04.go b/ddtrace/tracer/payload_v04.go index 8233951a19..90b59a8d7c 100644 --- a/ddtrace/tracer/payload_v04.go +++ b/ddtrace/tracer/payload_v04.go @@ -58,9 +58,6 @@ type payloadV04 struct { // reader is used for reading the contents of buf. reader *bytes.Reader - - // protocolVersion specifies the trace protocol to use. - protocolVersion float64 } var _ io.Reader = (*payloadV04)(nil) @@ -68,9 +65,8 @@ var _ io.Reader = (*payloadV04)(nil) // newPayloadV04 returns a ready to use payload. func newPayloadV04() *payloadV04 { p := &payloadV04{ - header: make([]byte, 8), - off: 8, - protocolVersion: traceProtocolV04, + header: make([]byte, 8), + off: 8, } return p } @@ -149,7 +145,7 @@ func (p *payloadV04) stats() payloadStats { // protocol returns the protocol version of the payload. func (p *payloadV04) protocol() float64 { - return p.protocolVersion + return traceProtocolV04 } // updateHeader updates the payload header based on the number of items currently diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 4c3ea68e3c..4d5c4ed60a 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -67,9 +67,6 @@ type payloadV1 struct { // a list of trace `chunks` chunks []traceChunk // 11 - // protocolVersion specifies the trace protocol to use. - protocolVersion float64 - // header specifies the first few bytes in the msgpack stream // indicating the type of map (fixmap, map16 or map32) // and the number of items contained in the stream. @@ -97,11 +94,10 @@ type payloadV1 struct { // newPayloadV1 returns a ready to use payloadV1. func newPayloadV1() *payloadV1 { return &payloadV1{ - protocolVersion: traceProtocolV1, - attributes: make(map[string]anyValue), - chunks: make([]traceChunk, 0), - readOff: 0, - writeOff: 0, + attributes: make(map[string]anyValue), + chunks: make([]traceChunk, 0), + readOff: 0, + writeOff: 0, } } @@ -220,7 +216,7 @@ func (p *payloadV1) itemCount() int { } func (p *payloadV1) protocol() float64 { - return p.protocolVersion + return traceProtocolV1 } func (p *payloadV1) updateHeader() { From bff64ad2226840c0b2626ab5a89c97e92cabc067 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Wed, 29 Oct 2025 13:55:25 -0400 Subject: [PATCH 72/75] nit: clarify docstring for itemcount() --- ddtrace/tracer/payload.go | 1 + 1 file changed, 1 insertion(+) diff --git a/ddtrace/tracer/payload.go b/ddtrace/tracer/payload.go index 8d8d04979e..8491189dc4 100644 --- a/ddtrace/tracer/payload.go +++ b/ddtrace/tracer/payload.go @@ -85,6 +85,7 @@ func (sp *safePayload) push(t spanList) (stats payloadStats, err error) { } // itemCount returns the number of items available in the stream in a thread-safe manner. +// This method is not thread-safe, but the underlying payload.itemCount() must be. func (sp *safePayload) itemCount() int { return sp.p.itemCount() } From c398de9c9d101b7ad826bc426bd895963103747b Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Wed, 29 Oct 2025 14:07:06 -0400 Subject: [PATCH 73/75] fix: incorrectly setting service on payload instead of chunks --- ddtrace/tracer/payload_test.go | 2 +- ddtrace/tracer/payload_v1.go | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/ddtrace/tracer/payload_test.go b/ddtrace/tracer/payload_test.go index 7c8fdfd430..30c11018ab 100644 --- a/ddtrace/tracer/payload_test.go +++ b/ddtrace/tracer/payload_test.go @@ -168,11 +168,11 @@ func TestPayloadV1Decode(t *testing.T) { assert.Empty(o) assert.NotEmpty(got.attributes) assert.Equal(p.attributes, got.attributes) - assert.Equal(got.attributes["service"].value, "golden") assert.Equal(got.attributes[keyProcessTags].value, processtags.GlobalTags().String()) assert.Greater(len(got.chunks), 0) assert.Equal(p.chunks[0].traceID, got.chunks[0].traceID) assert.Equal(p.chunks[0].spans[0].spanID, got.chunks[0].spans[0].spanID) + assert.Equal(got.chunks[0].attributes["service"].value, "golden") }) } } diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 4d5c4ed60a..538c8ce807 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -113,14 +113,15 @@ func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { // For now, we blindly set the origin, priority, and attributes values for the chunk // In the future, attributes should hold values that are shared across all chunks in the payload origin, priority, sm, traceID := "", 0, 0, [16]byte{} + attr := make(map[string]anyValue) for _, span := range t { if span == nil { continue } // If we haven't seen the service yet, we set it blindly assuming that all the spans created by // a service must share the same value. - if _, ok := p.attributes["service"]; !ok { - p.attributes["service"] = anyValue{valueType: StringValueType, value: span.Root().service} + if _, ok := attr["service"]; !ok { + attr["service"] = anyValue{valueType: StringValueType, value: span.Root().service} } binary.BigEndian.PutUint64(traceID[:8], span.Context().traceID.Upper()) binary.BigEndian.PutUint64(traceID[8:], span.Context().traceID.Lower()) @@ -146,14 +147,16 @@ func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { origin: origin, traceID: traceID[:], samplingMechanism: uint32(sm), + attributes: attr, } + // Append process tags to the payload attributes // if there are attributes available, set them in our bitmap and increment // the number of fields. + p.setProcessTags() if !p.bm.contains(10) && len(p.attributes) > 0 { p.bm.set(10) atomic.AddUint32(&p.fields, 1) - p.setProcessTags() } p.chunks = append(p.chunks, tc) From 5263a6f33bdba5ac1b35b49e437e29a8e6a6e849 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Wed, 29 Oct 2025 14:20:00 -0400 Subject: [PATCH 74/75] nit: improve docstring and var names --- ddtrace/tracer/payload_test.go | 6 ++++++ ddtrace/tracer/payload_v1.go | 6 +++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/ddtrace/tracer/payload_test.go b/ddtrace/tracer/payload_test.go index 30c11018ab..3dad5dd2fa 100644 --- a/ddtrace/tracer/payload_test.go +++ b/ddtrace/tracer/payload_test.go @@ -177,6 +177,9 @@ func TestPayloadV1Decode(t *testing.T) { } } +// TestPayloadV1EmbeddedStreamingStringTable tests that string values on the payload +// can be encoded and decoded correctly after using the string table. +// Tests repeated string values. func TestPayloadV1EmbeddedStreamingStringTable(t *testing.T) { p := newPayloadV1() p.SetHostname("production") @@ -200,6 +203,7 @@ func TestPayloadV1EmbeddedStreamingStringTable(t *testing.T) { assert.Equal(p.env, got.env) } +// TestPayloadV1UpdateHeader tests that the header of the payload is updated and grown correctly. func TestPayloadV1UpdateHeader(t *testing.T) { testCases := []uint32{ // Number of items 0, @@ -225,6 +229,8 @@ func TestPayloadV1UpdateHeader(t *testing.T) { } } +// TestEmptyPayloadV1 tests that an empty payload can be encoded and decoded correctly. +// Notably, it should send an empty map. func TestEmptyPayloadV1(t *testing.T) { p := newPayloadV1() assert := assert.New(t) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 538c8ce807..7ca1061d7e 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -126,9 +126,9 @@ func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { binary.BigEndian.PutUint64(traceID[:8], span.Context().traceID.Upper()) binary.BigEndian.PutUint64(traceID[8:], span.Context().traceID.Lower()) - if p, ok := span.Context().SamplingPriority(); ok { + if prio, ok := span.Context().SamplingPriority(); ok { origin = span.Context().origin // TODO(darccio): are we sure that origin will be shared across all the spans in the chunk? - priority = p // TODO(darccio): the same goes for priority. + priority = prio // TODO(darccio): the same goes for priority. dm := span.context.trace.propagatingTag(keyDecisionMaker) sm, err = strconv.Atoi(dm) if err != nil { @@ -309,7 +309,7 @@ type fieldValue interface { bool | []byte | int32 | int64 | uint32 | uint64 | string } -// encodeField takes a field of any value and encodes it into the given buffer +// encodeField takes a field of any fieldValue and encodes it into the given buffer // in msgp format. func encodeField[F fieldValue](buf []byte, bm bitmap, fieldID uint32, a F, st *stringTable) []byte { if !bm.contains(fieldID) { From 6e71dbdc803c5216df3d31f2080b6afabdc37298 Mon Sep 17 00:00:00 2001 From: Hannah Kim Date: Wed, 29 Oct 2025 14:38:12 -0400 Subject: [PATCH 75/75] fix: potential for going out of bounds of uint32 when using decision maker --- ddtrace/tracer/payload_v1.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/ddtrace/tracer/payload_v1.go b/ddtrace/tracer/payload_v1.go index 7ca1061d7e..adc4f89282 100644 --- a/ddtrace/tracer/payload_v1.go +++ b/ddtrace/tracer/payload_v1.go @@ -112,7 +112,7 @@ func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { // For now, we blindly set the origin, priority, and attributes values for the chunk // In the future, attributes should hold values that are shared across all chunks in the payload - origin, priority, sm, traceID := "", 0, 0, [16]byte{} + origin, priority, sm, traceID := "", 0, uint32(0), [16]byte{} attr := make(map[string]anyValue) for _, span := range t { if span == nil { @@ -130,17 +130,16 @@ func (p *payloadV1) push(t spanList) (stats payloadStats, err error) { origin = span.Context().origin // TODO(darccio): are we sure that origin will be shared across all the spans in the chunk? priority = prio // TODO(darccio): the same goes for priority. dm := span.context.trace.propagatingTag(keyDecisionMaker) - sm, err = strconv.Atoi(dm) - if err != nil { - log.Error("failed to convert decision maker to int: %s", err.Error()) + if v, err := strconv.ParseInt(dm, 10, 32); err == nil { + if v < 0 { + v = -v + } + sm = uint32(v) + } else { + log.Error("failed to convert decision maker to uint32: %s", err.Error()) } - break } } - // we want to represent sampling mechanism as a uint32, the absolute value of the decision maker - if sm < 0 { - sm *= -1 - } tc := traceChunk{ spans: t, priority: int32(priority),