Cardinality is the killer

1def ensure_stream(tenant, label_set):
2    stream_id = hash(sorted(label_set.items()))
3    if stream_id in active_streams[tenant]:
4        return active_streams[tenant][stream_id]
5    # new label-set -> new stream, new chunk, new index entry
6    if len(active_streams[tenant]) >= max_streams_per_user:
7        reject('per-stream limit exceeded')
8    chunk = open_chunk(stream_id)
9    active_streams[tenant][stream_id] = chunk
10    streams_created_total.inc()
11    return chunk

1def index_document(index, doc):
2    for key, value in flatten(doc).items():
3        if key not in mapping[index].fields:
4            if mapping[index].dynamic is False:
5                continue  # ignored, not indexed
6            if len(mapping[index].fields) >= 1000:
7                # index.mapping.total_fields.limit
8                raise MappingExplosion(index)
9            mapping[index].fields[key] = infer_type(value)
10        write_to_inverted_index(index, key, value)

1def relabel(entry):
2    # Low-cardinality dimensions stay as labels / mapped fields.
3    indexed = pick(entry, ['service', 'env', 'level', 'region'])
4    # High-cardinality dimensions move to the body /
5    # structured metadata, searchable by line filter only.
6    body = entry.body
7    body['request_id'] = entry.pop('request_id', None)
8    body['user_id']    = entry.pop('user_id', None)
9    body['trace_id']   = entry.pop('trace_id', None)
10    return { 'labels': indexed, 'body': body }