mirror of
https://github.com/OneUptime/oneuptime.git
synced 2026-01-11 19:56:44 +00:00
f49b1995df
- Add Telemetry service entrypoint - Telemetry/Index.ts: app bootstrap, routes mounting, infrastructure init and Telemetry SDK init. - Unified queue + worker - Telemetry/Jobs/TelemetryIngest/ProcessTelemetry.ts: single worker that dispatches queued jobs to specific processors (logs, traces, metrics, syslog, fluent logs). - Telemetry/Services/Queue/TelemetryQueueService.ts: central queue API and job payload types. - Per-type Queue wrappers (LogsQueueService, MetricsQueueService, TracesQueueService, FluentLogsQueueService, SyslogQueueService). - OpenTelemetry ingestion middleware and proto support - Telemetry/Middleware/OtelRequestMiddleware.ts: detect OTLP endpoint (logs/traces/metrics), decode protobuf bodies using protobufjs and set product type. - Telemetry/ProtoFiles/OTel/v1/*.proto: include common.proto, logs.proto, metrics.proto, resource.proto, traces.proto for OTLP v1 messages. - Ingest services - Telemetry/Services/OtelLogsIngestService.ts: parse incoming OTLP logs, map attributes, convert timestamps, batch insert logs. - Telemetry/Services/OtelTracesIngestService.ts: parse OTLP traces, build span rows, extract exceptions, batch insert spans and exceptions, save telemetry exception summary. - Telemetry/Services/OtelMetricsIngestService.ts: parse OTLP metrics, normalize datapoints, batch insert metrics and index metric name -> service map. - Telemetry/Services/SyslogIngestService.ts: syslog ingestion endpoints, parser integration, map syslog fields to attributes and logs. - Telemetry/Services/FluentLogsIngestService.ts: ingest Fluentd style logs, normalize entries and insert into log backend. - Telemetry/Services/OtelIngestBaseService.ts: helpers to resolve service name from attributes/headers. - Syslog parser and utilities - Telemetry/Utils/SyslogParser.ts: robust RFC5424 and RFC3164 parser, structured data extraction and sanitization. - Telemetry/Tests/Utils/SyslogParser.test.ts: unit tests for parser behavior. - Telemetry exception utilities - Telemetry/Utils/Exception.ts: generate exception fingerprint and upsert telemetry exception status (saveOrUpdateTelemetryException). - Queue & job integration - New integration with Common/Server/Infrastructure/Queue and QueueWorker, job id generation and telemetry job types. - Telemetry services add ingestion jobs instead of processing synchronously. - Config, build and dev tooling - Add Telemetry/package.json, package-lock.json, tsconfig.json, nodemon.json, jest config. - New script configs and dependencies (protobufjs, ts-node, jest, nodemon, etc). - Docker / environment updates - docker-compose.base.yml, docker-compose.dev.yml, docker-compose.yml: rename service from open-telemetry-ingest -> telemetry and wire TELEMETRY_* envs. - config.example.env: rename and consolidate environment variables (OPEN_TELEMETRY_* -> TELEMETRY_*, update hostnames and ports). - Tests/Scripts/status-check.sh: update ready-check target to telemetry/status/ready. - Other - Telemetry/Services/Queue/*: export helpers and legacy-compatible job interface shims. - Memory cleanup and batching safeguards across ingest services. - Logging and capture spans added to key code paths. BREAKING CHANGES / MIGRATION NOTES: - Environment variables and docker service names changed: - Replace OPEN_TELEMETRY_... vars with TELEMETRY_... (PORT, HOSTNAME, CONCURRENCY, DISABLE_TELEMETRY, etc). - docker-compose entries moved from "open-telemetry-ingest" to "telemetry" and image name changed to oneuptime/telemetry. - Update any deployment automation and monitoring checks referencing the old service name or endpoints. - Consumers: OTLP endpoints and behavior remain supported, but ingestion is now queued and processed asynchronously. Testing / Running: - Install deps in Telemetry/ (npm install) after syncing Common workspace. - Run dev: npx nodemon (nodemon.json) or build & start using provided scripts. - Run tests with jest (Telemetry test suite includes SyslogParser unit tests). Files added/modified (high level): - Added many files under Telemetry/: Index, Jobs, Middleware, ProtoFiles, Services, Utils, Tests, package and config artifacts. - Modified docker-compose.* and config.example.env and status check script to use new TELEMETRY service/vars.
475 lines
12 KiB
YAML
475 lines
12 KiB
YAML
name: Build
|
|
|
|
on:
|
|
pull_request:
|
|
push:
|
|
branches-ignore:
|
|
- 'hotfix-*'
|
|
- 'release'
|
|
|
|
|
|
jobs:
|
|
|
|
docker-build-accounts:
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
CI_PIPELINE_ID: ${{github.run_number}}
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Preinstall
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 10
|
|
max_attempts: 3
|
|
command: npm run prerun
|
|
|
|
# build image for accounts service
|
|
- name: build docker image
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 45
|
|
max_attempts: 3
|
|
command: sudo docker build --no-cache -f ./Accounts/Dockerfile .
|
|
|
|
docker-build-isolated-vm:
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
CI_PIPELINE_ID: ${{github.run_number}}
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Preinstall
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 10
|
|
max_attempts: 3
|
|
command: npm run prerun
|
|
|
|
# build image for accounts service
|
|
- name: build docker image
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 45
|
|
max_attempts: 3
|
|
command: sudo docker build --no-cache -f ./IsolatedVM/Dockerfile .
|
|
|
|
docker-build-home:
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
CI_PIPELINE_ID: ${{github.run_number}}
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Preinstall
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 10
|
|
max_attempts: 3
|
|
command: npm run prerun
|
|
|
|
# build image for accounts service
|
|
- name: build docker image
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 45
|
|
max_attempts: 3
|
|
command: sudo docker build --no-cache -f ./Home/Dockerfile .
|
|
|
|
docker-build-worker:
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
CI_PIPELINE_ID: ${{github.run_number}}
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Preinstall
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 10
|
|
max_attempts: 3
|
|
command: npm run prerun
|
|
|
|
# build image for accounts service
|
|
- name: build docker image
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 45
|
|
max_attempts: 3
|
|
command: sudo docker build --no-cache -f ./Worker/Dockerfile .
|
|
|
|
docker-build-workflow:
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
CI_PIPELINE_ID: ${{github.run_number}}
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Preinstall
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 10
|
|
max_attempts: 3
|
|
command: npm run prerun
|
|
|
|
# build image for accounts service
|
|
- name: build docker image
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 45
|
|
max_attempts: 3
|
|
command: sudo docker build --no-cache -f ./Workflow/Dockerfile .
|
|
|
|
docker-build-api-reference:
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
CI_PIPELINE_ID: ${{github.run_number}}
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Preinstall
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 10
|
|
max_attempts: 3
|
|
command: npm run prerun
|
|
|
|
# build image for accounts service
|
|
- name: build docker image
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 45
|
|
max_attempts: 3
|
|
command: sudo docker build --no-cache -f ./APIReference/Dockerfile .
|
|
|
|
docker-build-docs:
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
CI_PIPELINE_ID: ${{github.run_number}}
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Preinstall
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 10
|
|
max_attempts: 3
|
|
command: npm run prerun
|
|
|
|
# build image for accounts service
|
|
- name: build docker image
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 45
|
|
max_attempts: 3
|
|
command: sudo docker build --no-cache -f ./Docs/Dockerfile .
|
|
|
|
|
|
docker-build-otel-collector:
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
CI_PIPELINE_ID: ${{github.run_number}}
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Preinstall
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 10
|
|
max_attempts: 3
|
|
command: npm run prerun
|
|
|
|
# build image for accounts service
|
|
- name: build docker image
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 45
|
|
max_attempts: 3
|
|
command: sudo docker build --no-cache -f ./OTelCollector/Dockerfile .
|
|
|
|
docker-build-app:
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
CI_PIPELINE_ID: ${{github.run_number}}
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Preinstall
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 10
|
|
max_attempts: 3
|
|
command: npm run prerun
|
|
|
|
|
|
# build image for accounts service
|
|
- name: build docker image
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 45
|
|
max_attempts: 3
|
|
command: sudo docker build --no-cache -f ./App/Dockerfile .
|
|
|
|
|
|
docker-build-copilot:
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
CI_PIPELINE_ID: ${{github.run_number}}
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Preinstall
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 10
|
|
max_attempts: 3
|
|
command: npm run prerun
|
|
|
|
# build image for accounts service
|
|
- name: build docker image
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 45
|
|
max_attempts: 3
|
|
command: sudo docker build --no-cache -f ./Copilot/Dockerfile .
|
|
|
|
docker-build-e2e:
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
CI_PIPELINE_ID: ${{github.run_number}}
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Preinstall
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 10
|
|
max_attempts: 3
|
|
command: npm run prerun
|
|
|
|
|
|
# build image for accounts service
|
|
- name: build docker image
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 45
|
|
max_attempts: 3
|
|
command: sudo docker build --no-cache -f ./E2E/Dockerfile .
|
|
|
|
docker-build-admin-dashboard:
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
CI_PIPELINE_ID: ${{github.run_number}}
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Preinstall
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 10
|
|
max_attempts: 3
|
|
command: npm run prerun
|
|
|
|
# build image for home
|
|
- name: build docker image
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 45
|
|
max_attempts: 3
|
|
command: sudo docker build --no-cache -f ./AdminDashboard/Dockerfile .
|
|
|
|
docker-build-dashboard:
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
CI_PIPELINE_ID: ${{github.run_number}}
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Preinstall
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 10
|
|
max_attempts: 3
|
|
command: npm run prerun
|
|
|
|
# build image for home
|
|
- name: build docker image
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 45
|
|
max_attempts: 3
|
|
command: sudo docker build --no-cache -f ./Dashboard/Dockerfile .
|
|
|
|
docker-build-probe:
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
CI_PIPELINE_ID: ${{github.run_number}}
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Preinstall
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 10
|
|
max_attempts: 3
|
|
command: npm run prerun
|
|
|
|
# build image probe api
|
|
- name: build docker image
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 45
|
|
max_attempts: 3
|
|
command: sudo docker build --no-cache -f ./Probe/Dockerfile .
|
|
|
|
docker-build-probe-ingest:
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
CI_PIPELINE_ID: ${{github.run_number}}
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Preinstall
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 10
|
|
max_attempts: 3
|
|
command: npm run prerun
|
|
|
|
# build image probe api
|
|
- name: build docker image
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 45
|
|
max_attempts: 3
|
|
command: sudo docker build --no-cache -f ./ProbeIngest/Dockerfile .
|
|
|
|
docker-build-server-monitor-ingest:
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
CI_PIPELINE_ID: ${{github.run_number}}
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Preinstall
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 10
|
|
max_attempts: 3
|
|
command: npm run prerun
|
|
|
|
# build image probe api
|
|
- name: build docker image
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 45
|
|
max_attempts: 3
|
|
command: sudo docker build --no-cache -f ./ServerMonitorIngest/Dockerfile .
|
|
|
|
docker-build-telemetry:
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
CI_PIPELINE_ID: ${{github.run_number}}
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Preinstall
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 10
|
|
max_attempts: 3
|
|
command: npm run prerun
|
|
|
|
# build image probe api
|
|
- name: build docker image
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 45
|
|
max_attempts: 3
|
|
command: sudo docker build --no-cache -f ./Telemetry/Dockerfile .
|
|
|
|
docker-build-incoming-request-ingest:
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
CI_PIPELINE_ID: ${{github.run_number}}
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Preinstall
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 10
|
|
max_attempts: 3
|
|
command: npm run prerun
|
|
|
|
# build image probe api
|
|
- name: build docker image
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 45
|
|
max_attempts: 3
|
|
command: sudo docker build --no-cache -f ./IncomingRequestIngest/Dockerfile .
|
|
|
|
docker-build-status-page:
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
CI_PIPELINE_ID: ${{github.run_number}}
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Preinstall
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 10
|
|
max_attempts: 3
|
|
command: npm run prerun
|
|
|
|
# build image for home
|
|
- name: build docker image
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 45
|
|
max_attempts: 3
|
|
command: sudo docker build --no-cache -f ./StatusPage/Dockerfile .
|
|
|
|
docker-build-test-server:
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
CI_PIPELINE_ID: ${{github.run_number}}
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Preinstall
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 10
|
|
max_attempts: 3
|
|
command: npm run prerun
|
|
|
|
# build image for mail service
|
|
- name: build docker image
|
|
uses: nick-fields/retry@v3
|
|
with:
|
|
timeout_minutes: 45
|
|
max_attempts: 3
|
|
command: sudo docker build --no-cache -f ./TestServer/Dockerfile .
|