mirror of
https://github.com/OneUptime/oneuptime.git
synced 2026-01-11 19:56:44 +00:00
f49b1995df
- Add Telemetry service entrypoint - Telemetry/Index.ts: app bootstrap, routes mounting, infrastructure init and Telemetry SDK init. - Unified queue + worker - Telemetry/Jobs/TelemetryIngest/ProcessTelemetry.ts: single worker that dispatches queued jobs to specific processors (logs, traces, metrics, syslog, fluent logs). - Telemetry/Services/Queue/TelemetryQueueService.ts: central queue API and job payload types. - Per-type Queue wrappers (LogsQueueService, MetricsQueueService, TracesQueueService, FluentLogsQueueService, SyslogQueueService). - OpenTelemetry ingestion middleware and proto support - Telemetry/Middleware/OtelRequestMiddleware.ts: detect OTLP endpoint (logs/traces/metrics), decode protobuf bodies using protobufjs and set product type. - Telemetry/ProtoFiles/OTel/v1/*.proto: include common.proto, logs.proto, metrics.proto, resource.proto, traces.proto for OTLP v1 messages. - Ingest services - Telemetry/Services/OtelLogsIngestService.ts: parse incoming OTLP logs, map attributes, convert timestamps, batch insert logs. - Telemetry/Services/OtelTracesIngestService.ts: parse OTLP traces, build span rows, extract exceptions, batch insert spans and exceptions, save telemetry exception summary. - Telemetry/Services/OtelMetricsIngestService.ts: parse OTLP metrics, normalize datapoints, batch insert metrics and index metric name -> service map. - Telemetry/Services/SyslogIngestService.ts: syslog ingestion endpoints, parser integration, map syslog fields to attributes and logs. - Telemetry/Services/FluentLogsIngestService.ts: ingest Fluentd style logs, normalize entries and insert into log backend. - Telemetry/Services/OtelIngestBaseService.ts: helpers to resolve service name from attributes/headers. - Syslog parser and utilities - Telemetry/Utils/SyslogParser.ts: robust RFC5424 and RFC3164 parser, structured data extraction and sanitization. - Telemetry/Tests/Utils/SyslogParser.test.ts: unit tests for parser behavior. - Telemetry exception utilities - Telemetry/Utils/Exception.ts: generate exception fingerprint and upsert telemetry exception status (saveOrUpdateTelemetryException). - Queue & job integration - New integration with Common/Server/Infrastructure/Queue and QueueWorker, job id generation and telemetry job types. - Telemetry services add ingestion jobs instead of processing synchronously. - Config, build and dev tooling - Add Telemetry/package.json, package-lock.json, tsconfig.json, nodemon.json, jest config. - New script configs and dependencies (protobufjs, ts-node, jest, nodemon, etc). - Docker / environment updates - docker-compose.base.yml, docker-compose.dev.yml, docker-compose.yml: rename service from open-telemetry-ingest -> telemetry and wire TELEMETRY_* envs. - config.example.env: rename and consolidate environment variables (OPEN_TELEMETRY_* -> TELEMETRY_*, update hostnames and ports). - Tests/Scripts/status-check.sh: update ready-check target to telemetry/status/ready. - Other - Telemetry/Services/Queue/*: export helpers and legacy-compatible job interface shims. - Memory cleanup and batching safeguards across ingest services. - Logging and capture spans added to key code paths. BREAKING CHANGES / MIGRATION NOTES: - Environment variables and docker service names changed: - Replace OPEN_TELEMETRY_... vars with TELEMETRY_... (PORT, HOSTNAME, CONCURRENCY, DISABLE_TELEMETRY, etc). - docker-compose entries moved from "open-telemetry-ingest" to "telemetry" and image name changed to oneuptime/telemetry. - Update any deployment automation and monitoring checks referencing the old service name or endpoints. - Consumers: OTLP endpoints and behavior remain supported, but ingestion is now queued and processed asynchronously. Testing / Running: - Install deps in Telemetry/ (npm install) after syncing Common workspace. - Run dev: npx nodemon (nodemon.json) or build & start using provided scripts. - Run tests with jest (Telemetry test suite includes SyslogParser unit tests). Files added/modified (high level): - Added many files under Telemetry/: Index, Jobs, Middleware, ProtoFiles, Services, Utils, Tests, package and config artifacts. - Modified docker-compose.* and config.example.env and status check script to use new TELEMETRY service/vars.
480 lines
No EOL
15 KiB
YAML
480 lines
No EOL
15 KiB
YAML
|
|
|
|
services:
|
|
|
|
|
|
redis:
|
|
ports:
|
|
- '6310:6379'
|
|
extends:
|
|
file: ./docker-compose.base.yml
|
|
service: redis
|
|
|
|
clickhouse:
|
|
ports:
|
|
- '9034:9000'
|
|
- '8189:8123'
|
|
extends:
|
|
file: ./docker-compose.base.yml
|
|
service: clickhouse
|
|
volumes:
|
|
- ./Clickhouse/config.xml:/etc/clickhouse-server/config.xml
|
|
|
|
postgres:
|
|
ports:
|
|
- '5400:5432'
|
|
extends:
|
|
file: ./docker-compose.base.yml
|
|
service: postgres
|
|
|
|
|
|
otel-collector:
|
|
extends:
|
|
file: ./docker-compose.base.yml
|
|
service: otel-collector
|
|
build:
|
|
network: host
|
|
context: .
|
|
dockerfile: ./OTelCollector/Dockerfile
|
|
|
|
accounts:
|
|
ports:
|
|
- '${ACCOUNTS_PORT}:${ACCOUNTS_PORT}'
|
|
extends:
|
|
file: ./docker-compose.base.yml
|
|
service: accounts
|
|
volumes:
|
|
- ./Accounts:/usr/src/app:cached
|
|
# Use node modules of the container and not host system.
|
|
# https://stackoverflow.com/questions/29181032/add-a-volume-to-docker-but-exclude-a-sub-folder
|
|
- /usr/src/app/dev-env
|
|
- /usr/src/app/node_modules/
|
|
- ./Common:/usr/src/Common:cached
|
|
|
|
- /usr/src/Common/node_modules/
|
|
|
|
|
|
build:
|
|
network: host
|
|
context: .
|
|
dockerfile: ./Accounts/Dockerfile
|
|
|
|
|
|
dashboard:
|
|
ports:
|
|
- '${DASHBOARD_PORT}:${DASHBOARD_PORT}'
|
|
extends:
|
|
file: ./docker-compose.base.yml
|
|
service: dashboard
|
|
volumes:
|
|
- ./Dashboard:/usr/src/app:cached
|
|
# Use node modules of the container and not host system.
|
|
# https://stackoverflow.com/questions/29181032/add-a-volume-to-docker-but-exclude-a-sub-folder
|
|
- /usr/src/app/dev-env
|
|
- /usr/src/app/node_modules/
|
|
- ./Common:/usr/src/Common:cached
|
|
|
|
- /usr/src/Common/node_modules/
|
|
|
|
|
|
build:
|
|
network: host
|
|
context: .
|
|
dockerfile: ./Dashboard/Dockerfile
|
|
|
|
|
|
|
|
admin-dashboard:
|
|
ports:
|
|
- '${ADMIN_DASHBOARD_PORT}:${ADMIN_DASHBOARD_PORT}'
|
|
extends:
|
|
file: ./docker-compose.base.yml
|
|
service: admin-dashboard
|
|
volumes:
|
|
- ./AdminDashboard:/usr/src/app:cached
|
|
# Use node modules of the container and not host system.
|
|
# https://stackoverflow.com/questions/29181032/add-a-volume-to-docker-but-exclude-a-sub-folder
|
|
- /usr/src/app/dev-env
|
|
- /usr/src/app/node_modules/
|
|
- ./Common:/usr/src/Common:cached
|
|
|
|
- /usr/src/Common/node_modules/
|
|
|
|
|
|
build:
|
|
network: host
|
|
context: .
|
|
dockerfile: ./AdminDashboard/Dockerfile
|
|
|
|
|
|
status-page:
|
|
ports:
|
|
- '${STATUS_PAGE_PORT}:${STATUS_PAGE_PORT}'
|
|
extends:
|
|
file: ./docker-compose.base.yml
|
|
service: status-page
|
|
volumes:
|
|
- ./StatusPage:/usr/src/app:cached
|
|
# Use node modules of the container and not host system.
|
|
# https://stackoverflow.com/questions/29181032/add-a-volume-to-docker-but-exclude-a-sub-folder
|
|
- /usr/src/app/dev-env
|
|
- /usr/src/app/node_modules/
|
|
- ./Common:/usr/src/Common:cached
|
|
|
|
- /usr/src/Common/node_modules/
|
|
|
|
|
|
build:
|
|
network: host
|
|
context: .
|
|
dockerfile: ./StatusPage/Dockerfile
|
|
|
|
|
|
test-server:
|
|
volumes:
|
|
- ./TestServer:/usr/src/app:cached
|
|
# Use node modules of the container and not host system.
|
|
# https://stackoverflow.com/questions/29181032/add-a-volume-to-docker-but-exclude-a-sub-folder
|
|
- /usr/src/app/node_modules/
|
|
- ./Common:/usr/src/Common:cached
|
|
|
|
- /usr/src/Common/node_modules/
|
|
|
|
|
|
extends:
|
|
file: ./docker-compose.base.yml
|
|
service: test-server
|
|
ports:
|
|
- '9141:9229' # Debugging port.
|
|
- '3800:3800'
|
|
build:
|
|
network: host
|
|
context: .
|
|
dockerfile: ./TestServer/Dockerfile
|
|
|
|
|
|
home:
|
|
volumes:
|
|
- ./Home:/usr/src/app:cached
|
|
# Use node modules of the container and not host system.
|
|
# https://stackoverflow.com/questions/29181032/add-a-volume-to-docker-but-exclude-a-sub-folder
|
|
- /usr/src/app/node_modules/
|
|
- ./Common:/usr/src/Common:cached
|
|
- /usr/src/Common/node_modules/
|
|
extends:
|
|
file: ./docker-compose.base.yml
|
|
service: home
|
|
ports:
|
|
- '9212:9229' # Debugging port.
|
|
build:
|
|
network: host
|
|
context: .
|
|
dockerfile: ./Home/Dockerfile
|
|
|
|
worker:
|
|
volumes:
|
|
- ./Worker:/usr/src/app:cached
|
|
# Use node modules of the container and not host system.
|
|
# https://stackoverflow.com/questions/29181032/add-a-volume-to-docker-but-exclude-a-sub-folder
|
|
- /usr/src/app/node_modules/
|
|
- ./Common:/usr/src/Common:cached
|
|
- /usr/src/Common/node_modules/
|
|
extends:
|
|
file: ./docker-compose.base.yml
|
|
service: worker
|
|
ports:
|
|
- '8734:9229' # Debugging port.
|
|
build:
|
|
network: host
|
|
context: .
|
|
dockerfile: ./Worker/Dockerfile
|
|
|
|
|
|
workflow:
|
|
volumes:
|
|
- ./Workflow:/usr/src/app:cached
|
|
# Use node modules of the container and not host system.
|
|
# https://stackoverflow.com/questions/29181032/add-a-volume-to-docker-but-exclude-a-sub-folder
|
|
- /usr/src/app/node_modules/
|
|
- ./Common:/usr/src/Common:cached
|
|
- /usr/src/Common/node_modules/
|
|
extends:
|
|
file: ./docker-compose.base.yml
|
|
service: workflow
|
|
ports:
|
|
- '8735:9229' # Debugging port.
|
|
build:
|
|
network: host
|
|
context: .
|
|
dockerfile: ./Workflow/Dockerfile
|
|
|
|
api-reference:
|
|
volumes:
|
|
- ./APIReference:/usr/src/app:cached
|
|
# Use node modules of the container and not host system.
|
|
# https://stackoverflow.com/questions/29181032/add-a-volume-to-docker-but-exclude-a-sub-folder
|
|
- /usr/src/app/node_modules/
|
|
- ./Common:/usr/src/Common:cached
|
|
- /usr/src/Common/node_modules/
|
|
extends:
|
|
file: ./docker-compose.base.yml
|
|
service: api-reference
|
|
ports:
|
|
- '8737:9229' # Debugging port.
|
|
build:
|
|
network: host
|
|
context: .
|
|
dockerfile: ./APIReference/Dockerfile
|
|
|
|
docs:
|
|
volumes:
|
|
- ./Docs:/usr/src/app:cached
|
|
# Use node modules of the container and not host system.
|
|
# https://stackoverflow.com/questions/29181032/add-a-volume-to-docker-but-exclude-a-sub-folder
|
|
- /usr/src/app/node_modules/
|
|
- ./Common:/usr/src/Common:cached
|
|
- /usr/src/Common/node_modules/
|
|
extends:
|
|
file: ./docker-compose.base.yml
|
|
service: docs
|
|
ports:
|
|
- '8738:9229' # Debugging port.
|
|
build:
|
|
network: host
|
|
context: .
|
|
dockerfile: ./Docs/Dockerfile
|
|
|
|
app:
|
|
volumes:
|
|
- ./App:/usr/src/app:cached
|
|
# Use node modules of the container and not host system.
|
|
# https://stackoverflow.com/questions/29181032/add-a-volume-to-docker-but-exclude-a-sub-folder
|
|
- /usr/src/app/node_modules/
|
|
- ./Common:/usr/src/Common:cached
|
|
- /usr/src/Common/node_modules/
|
|
extends:
|
|
file: ./docker-compose.base.yml
|
|
service: app
|
|
ports:
|
|
- '9232:9229' # Debugging port.
|
|
build:
|
|
network: host
|
|
context: .
|
|
dockerfile: ./App/Dockerfile
|
|
|
|
|
|
probe-1:
|
|
volumes:
|
|
- ./Probe:/usr/src/app:cached
|
|
# Use node modules of the container and not host system.
|
|
# https://stackoverflow.com/questions/29181032/add-a-volume-to-docker-but-exclude-a-sub-folder
|
|
- /usr/src/app/node_modules/
|
|
- ./Common:/usr/src/Common:cached
|
|
|
|
- /usr/src/Common/node_modules/
|
|
|
|
|
|
extends:
|
|
file: ./docker-compose.base.yml
|
|
service: probe-1
|
|
build:
|
|
network: host
|
|
context: .
|
|
dockerfile: ./Probe/Dockerfile
|
|
|
|
probe-2:
|
|
volumes:
|
|
- ./Probe:/usr/src/app:cached
|
|
# Use node modules of the container and not host system.
|
|
# https://stackoverflow.com/questions/29181032/add-a-volume-to-docker-but-exclude-a-sub-folder
|
|
- /usr/src/app/node_modules/
|
|
- ./Common:/usr/src/Common:cached
|
|
|
|
- /usr/src/Common/node_modules/
|
|
|
|
|
|
extends:
|
|
file: ./docker-compose.base.yml
|
|
service: probe-2
|
|
build:
|
|
network: host
|
|
context: .
|
|
dockerfile: ./Probe/Dockerfile
|
|
|
|
isolated-vm:
|
|
volumes:
|
|
- ./IsolatedVM:/usr/src/app:cached
|
|
# Use node modules of the container and not host system.
|
|
# https://stackoverflow.com/questions/29181032/add-a-volume-to-docker-but-exclude-a-sub-folder
|
|
- /usr/src/app/node_modules/
|
|
- ./Common:/usr/src/Common:cached
|
|
|
|
- /usr/src/Common/node_modules/
|
|
|
|
|
|
ports:
|
|
- '9974:9229' # Debugging port.
|
|
extends:
|
|
file: ./docker-compose.base.yml
|
|
service: isolated-vm
|
|
build:
|
|
network: host
|
|
context: .
|
|
dockerfile: ./IsolatedVM/Dockerfile
|
|
|
|
probe-ingest:
|
|
volumes:
|
|
- ./ProbeIngest:/usr/src/app:cached
|
|
# Use node modules of the container and not host system.
|
|
# https://stackoverflow.com/questions/29181032/add-a-volume-to-docker-but-exclude-a-sub-folder
|
|
- /usr/src/app/node_modules/
|
|
- ./Common:/usr/src/Common:cached
|
|
- /usr/src/Common/node_modules/
|
|
ports:
|
|
- '9932:9229' # Debugging port.
|
|
extends:
|
|
file: ./docker-compose.base.yml
|
|
service: probe-ingest
|
|
build:
|
|
network: host
|
|
context: .
|
|
dockerfile: ./ProbeIngest/Dockerfile
|
|
|
|
server-monitor-ingest:
|
|
volumes:
|
|
- ./ServerMonitorIngest:/usr/src/app:cached
|
|
# Use node modules of the container and not host system.
|
|
# https://stackoverflow.com/questions/29181032/add-a-volume-to-docker-but-exclude-a-sub-folder
|
|
- /usr/src/app/node_modules/
|
|
- ./Common:/usr/src/Common:cached
|
|
- /usr/src/Common/node_modules/
|
|
ports:
|
|
- '9941:9229' # Debugging port.
|
|
extends:
|
|
file: ./docker-compose.base.yml
|
|
service: server-monitor-ingest
|
|
build:
|
|
network: host
|
|
context: .
|
|
dockerfile: ./ServerMonitorIngest/Dockerfile
|
|
|
|
telemetry:
|
|
volumes:
|
|
- ./Telemetry:/usr/src/app:cached
|
|
# Use node modules of the container and not host system.
|
|
# https://stackoverflow.com/questions/29181032/add-a-volume-to-docker-but-exclude-a-sub-folder
|
|
- /usr/src/app/node_modules/
|
|
- ./Common:/usr/src/Common:cached
|
|
- /usr/src/Common/node_modules/
|
|
ports:
|
|
- '9938:9229' # Debugging port.
|
|
extends:
|
|
file: ./docker-compose.base.yml
|
|
service: telemetry
|
|
build:
|
|
network: host
|
|
context: .
|
|
dockerfile: ./Telemetry/Dockerfile
|
|
|
|
incoming-request-ingest:
|
|
volumes:
|
|
- ./IncomingRequestIngest:/usr/src/app:cached
|
|
# Use node modules of the container and not host system.
|
|
# https://stackoverflow.com/questions/29181032/add-a-volume-to-docker-but-exclude-a-sub-folder
|
|
- /usr/src/app/node_modules/
|
|
- ./Common:/usr/src/Common:cached
|
|
- /usr/src/Common/node_modules/
|
|
ports:
|
|
- '9933:9229' # Debugging port.
|
|
extends:
|
|
file: ./docker-compose.base.yml
|
|
service: incoming-request-ingest
|
|
build:
|
|
network: host
|
|
context: .
|
|
dockerfile: ./IncomingRequestIngest/Dockerfile
|
|
|
|
|
|
|
|
# Fluentd. Required only for development. In production its the responsibility of the customer to run fluentd and pipe logs to OneUptime.
|
|
# We run this container just for development, to see if logs are piped.
|
|
|
|
fluentd:
|
|
ports:
|
|
- 24224:24224
|
|
- 24224:24224/udp
|
|
- 8888:8888
|
|
user: fluent
|
|
extends:
|
|
file: ./docker-compose.base.yml
|
|
service: fluentd
|
|
volumes:
|
|
- ./Fluentd/fluent.conf:/fluentd/etc/fluent.conf
|
|
build:
|
|
network: host
|
|
context: ./Fluentd
|
|
dockerfile: ./Dockerfile
|
|
|
|
fluent-bit:
|
|
ports:
|
|
- 24225:24224
|
|
- 24285:24284
|
|
- 2020:2020
|
|
- 8889:8889
|
|
extends:
|
|
file: ./docker-compose.base.yml
|
|
service: fluent-bit
|
|
volumes:
|
|
- ./FluentBit/etc:/fluent-bit/etc/
|
|
build:
|
|
network: host
|
|
context: .
|
|
dockerfile: ./FluentBit/Dockerfile
|
|
|
|
ingress:
|
|
build:
|
|
network: host
|
|
context: .
|
|
dockerfile: ./Nginx/Dockerfile
|
|
extends:
|
|
file: ./docker-compose.base.yml
|
|
service: ingress
|
|
|
|
# e2e tests
|
|
e2e:
|
|
extends:
|
|
file: ./docker-compose.base.yml
|
|
service: e2e
|
|
volumes:
|
|
- ./E2E/playwright-report:/usr/src/app/playwright-report
|
|
- ./E2E/test-results:/usr/src/app/test-results
|
|
build:
|
|
network: host
|
|
context: .
|
|
dockerfile: ./E2E/Dockerfile
|
|
|
|
copilot:
|
|
volumes:
|
|
- ./Copilot:/usr/src/app:cached
|
|
# Use node modules of the container and not host system.
|
|
# https://stackoverflow.com/questions/29181032/add-a-volume-to-docker-but-exclude-a-sub-folder
|
|
- /usr/src/app/node_modules/
|
|
- ./Common:/usr/src/Common:cached
|
|
- /usr/src/Common/node_modules/
|
|
ports:
|
|
- '9985:9229' # Debugging port.
|
|
extends:
|
|
file: ./docker-compose.base.yml
|
|
service: copilot
|
|
build:
|
|
network: host
|
|
context: .
|
|
dockerfile: ./Copilot/Dockerfile
|
|
|
|
volumes:
|
|
postgres:
|
|
clickhouse:
|
|
|
|
networks:
|
|
oneuptime:
|
|
driver: bridge |