Neblic Collector¶

The Neblic collector is based on the OpenTelemetry collector. This page shows an example configuration file for an OpenTelemetry collector built with the Neblic extension.

The options that have a value replaceable with an environment variable, ${env:VAR_NAME}, are intended to be configured when using the container distribution of the collector.

receivers:
  otlp/receiver:
    protocols:
      grpc:
        # Required: Data plane endpoint.
        endpoint: "${env:NEBLIC_DATA_PLANE_RECEIVER_ENDPOINT}"

        # Uncomment to enable TLS
        # tls:
        #   cert_file: /etc/otelcol/ca/otelcol.crt
        #   key_file: /etc/otelcol/ca/otelcol.key

        # To use Bearer authentication TLS needs to be enabled
        # auth:
        #   authenticator: bearerauth

processors:
  batch:
    send_batch_size: "${env:PROCESSOR_BATCH_SEND_BATCH_SIZE}"
    timeout: "${env:PROCESSOR_BATCH_TIMEOUT}"
    send_batch_max_size: "${env:PROCESSOR_BATCH_SEND_BATCH_MAX_SIZE}"
    metadata_keys: "${env:PROCESSOR_BATCH_METADATA_KEYS}"
    metadata_cardinality_limit: "${env:PROCESSOR_BATCH_METADATA_CARDINALITY_LIMIT}"

# --8<-- [start:LokiProcessor]
  transform/propagate_loki_labels:
    error_mode: ignore
    log_statements:
    - context: log
      statements:
        - set(attributes["sampler"], instrumentation_scope.name)
        - set(attributes["resource"], resource.attributes["service.name"])
        - set(attributes["loki.attribute.labels"], ["sampler", "resource", "com.neblic.sample.type", "com.neblic.sample.stream.uids", "com.neblic.sample.encoding"])
        - set(attributes["loki.format"], "raw")
# --8<-- [end:LokiProcessor]

  filter/remove_raw_samples:
    error_mode: ignore
    logs:
      log_record:
        - attributes["com.neblic.sample.type"] == "raw"

connectors:
  neblic:
    # Collector id, if not provided, a random uid is used
    # uid: <unique collector id>

    # Required: Control Plane endpoint
    endpoint: "${env:NEBLIC_CONTROL_PLANE_ENDPOINT}"

    # Where will be configurations persisted on disk. If unset, configurations will be only kept in memory and lost when the collector restarts
    storage_path: "${env:NEBLIC_STORAGE_PATH}"

    # Uncomment to enable TLS
    # tls:
    #   cert_file: /etc/otelcol/ca/otelcol.crt
    #   key_file: /etc/otelcol/ca/otelcol.key

    # To use Bearer authentication TLS needs to be enabled
    # auth:
    #  type: bearer
    #  bearer:
    #     token: some_secret_token

exporters:
  # Uncomment to enable simple summary messages with the amount of `Data Samples` exported
  # logging:
  #   verbosity: normal

# --8<-- [start:LokiExporter]
  loki:
    endpoint: ${env:NEBLIC_DATA_PLANE_EXPORTER_LOKI_ENDPOINT}
    default_labels_enabled:
      exporter: false
      job: false
      instance: false
      level: false
# --8<-- [end:LokiExporter]

  otlp/cloud:
    endpoint: "${env:NEBLIC_DATA_PLANE_EXPORTER_ENDPOINT}"
    tls:
      insecure: false
    auth:
      authenticator: bearertokenauth

extensions:
  health_check:
    endpoint: 0.0.0.0:5000
  bearertokenauth:
    scheme: Bearer
    token: "${env:NEBLIC_DATA_PLANE_EXPORTER_BEARER_TOKEN}"

service:
  # Collector configuration
  telemetry:
    logs:
      level: info
      encoding: json

  pipelines:
    logs/input:
      receivers:
      - otlp/receiver
      processors:
      - batch
      exporters:
      - neblic
    logs/output_cloud:
      receivers:
      - neblic
      processors:
      - filter/remove_raw_samples
      exporters:
      - otlp/cloud
# --8<-- [start:LokiPipeline]
    logs/loki:
      receivers:
      - neblic
      processors:
      - transform/propagate_loki_labels
      exporters:
      - loki
# --8<-- [end:LokiPipeline]
  extensions:
    - health_check
    - bearertokenauth

The entrypoint.sh file defines the default values for the environment variables.

#!/bin/sh
set -euo pipefail

# set default configuration settings
OTELCOL_CONFIG_PATH="${OTELCOL_CONFIG_PATH:-/etc/neblic/otelcol/config.yaml}"

# Bach processor
export PROCESSOR_BATCH_SEND_BATCH_SIZE="${PROCESSOR_BATCH_SEND_BATCH_SIZE:-8192}"
export PROCESSOR_BATCH_TIMEOUT="${PROCESSOR_BATCH_TIMEOUT:-200ms}"
export PROCESSOR_BATCH_SEND_BATCH_MAX_SIZE="${PROCESSOR_BATCH_SEND_BATCH_MAX_SIZE:-0}"
export PROCESSOR_BATCH_METADATA_KEYS="${PROCESSOR_BATCH_METADATA_KEYS:-}"
export PROCESSOR_BATCH_METADATA_CARDINALITY_LIMIT="${PROCESSOR_BATCH_METADATA_CARDINALITY_LIMIT:-1000}"

# Neblic
export NEBLIC_DATA_PLANE_RECEIVER_ENDPOINT="${NEBLIC_DATA_PLANE_ENDPOINT:-0.0.0.0:4317}"
export NEBLIC_DATA_PLANE_EXPORTER_ENDPOINT="${NEBLIC_DATA_PLANE_EXPORTER_ENDPOINT:-gate.neblic.com:443}"
export NEBLIC_DATA_PLANE_EXPORTER_BEARER_TOKEN="${NEBLIC_DATA_PLANE_EXPORTER_BEARER_TOKEN:-}"
export NEBLIC_CONTROL_PLANE_ENDPOINT="${NEBLIC_CONTROL_PLANE_ENDPOINT:-0.0.0.0:8899}"
export NEBLIC_STORAGE_PATH="${NEBLIC_STORAGE_PATH:-/var/lib/otelcol/storage.yml}"

[[ ! -z "$NEBLIC_STORAGE_PATH" ]] &&
    echo "Setting Neblic storage path ${NEBLIC_STORAGE_PATH}" &&
    mkdir -p $NEBLIC_STORAGE_PATH &&
    chown -R otelcol $NEBLIC_STORAGE_PATH

echo "Starting otelcol"
exec /usr/bin/sudo -E -u otelcol /bin/otelcol --config $OTELCOL_CONFIG_PATH