From f49b1995dfcc3f77291b5178d30df17448e2db2b Mon Sep 17 00:00:00 2001 From: Nawaz Dhandala Date: Fri, 7 Nov 2025 21:36:47 +0000 Subject: [PATCH] feat(telemetry): add new Telemetry service (OTel, Syslog, Fluent, Metrics, Traces) and unified ingestion pipeline - Add Telemetry service entrypoint - Telemetry/Index.ts: app bootstrap, routes mounting, infrastructure init and Telemetry SDK init. - Unified queue + worker - Telemetry/Jobs/TelemetryIngest/ProcessTelemetry.ts: single worker that dispatches queued jobs to specific processors (logs, traces, metrics, syslog, fluent logs). - Telemetry/Services/Queue/TelemetryQueueService.ts: central queue API and job payload types. - Per-type Queue wrappers (LogsQueueService, MetricsQueueService, TracesQueueService, FluentLogsQueueService, SyslogQueueService). - OpenTelemetry ingestion middleware and proto support - Telemetry/Middleware/OtelRequestMiddleware.ts: detect OTLP endpoint (logs/traces/metrics), decode protobuf bodies using protobufjs and set product type. - Telemetry/ProtoFiles/OTel/v1/*.proto: include common.proto, logs.proto, metrics.proto, resource.proto, traces.proto for OTLP v1 messages. - Ingest services - Telemetry/Services/OtelLogsIngestService.ts: parse incoming OTLP logs, map attributes, convert timestamps, batch insert logs. - Telemetry/Services/OtelTracesIngestService.ts: parse OTLP traces, build span rows, extract exceptions, batch insert spans and exceptions, save telemetry exception summary. - Telemetry/Services/OtelMetricsIngestService.ts: parse OTLP metrics, normalize datapoints, batch insert metrics and index metric name -> service map. - Telemetry/Services/SyslogIngestService.ts: syslog ingestion endpoints, parser integration, map syslog fields to attributes and logs. - Telemetry/Services/FluentLogsIngestService.ts: ingest Fluentd style logs, normalize entries and insert into log backend. - Telemetry/Services/OtelIngestBaseService.ts: helpers to resolve service name from attributes/headers. - Syslog parser and utilities - Telemetry/Utils/SyslogParser.ts: robust RFC5424 and RFC3164 parser, structured data extraction and sanitization. - Telemetry/Tests/Utils/SyslogParser.test.ts: unit tests for parser behavior. - Telemetry exception utilities - Telemetry/Utils/Exception.ts: generate exception fingerprint and upsert telemetry exception status (saveOrUpdateTelemetryException). - Queue & job integration - New integration with Common/Server/Infrastructure/Queue and QueueWorker, job id generation and telemetry job types. - Telemetry services add ingestion jobs instead of processing synchronously. - Config, build and dev tooling - Add Telemetry/package.json, package-lock.json, tsconfig.json, nodemon.json, jest config. - New script configs and dependencies (protobufjs, ts-node, jest, nodemon, etc). - Docker / environment updates - docker-compose.base.yml, docker-compose.dev.yml, docker-compose.yml: rename service from open-telemetry-ingest -> telemetry and wire TELEMETRY_* envs. - config.example.env: rename and consolidate environment variables (OPEN_TELEMETRY_* -> TELEMETRY_*, update hostnames and ports). - Tests/Scripts/status-check.sh: update ready-check target to telemetry/status/ready. - Other - Telemetry/Services/Queue/*: export helpers and legacy-compatible job interface shims. - Memory cleanup and batching safeguards across ingest services. - Logging and capture spans added to key code paths. BREAKING CHANGES / MIGRATION NOTES: - Environment variables and docker service names changed: - Replace OPEN_TELEMETRY_... vars with TELEMETRY_... (PORT, HOSTNAME, CONCURRENCY, DISABLE_TELEMETRY, etc). - docker-compose entries moved from "open-telemetry-ingest" to "telemetry" and image name changed to oneuptime/telemetry. - Update any deployment automation and monitoring checks referencing the old service name or endpoints. - Consumers: OTLP endpoints and behavior remain supported, but ingestion is now queued and processed asynchronously. Testing / Running: - Install deps in Telemetry/ (npm install) after syncing Common workspace. - Run dev: npx nodemon (nodemon.json) or build & start using provided scripts. - Run tests with jest (Telemetry test suite includes SyslogParser unit tests). Files added/modified (high level): - Added many files under Telemetry/: Index, Jobs, Middleware, ProtoFiles, Services, Utils, Tests, package and config artifacts. - Modified docker-compose.* and config.example.env and status check script to use new TELEMETRY service/vars. --- .github/workflows/build.yml | 4 +- .github/workflows/compile.yml | 6 +- .github/workflows/release.yml | 28 ++++---- .github/workflows/test-release.yaml | 28 ++++---- ...emetry-ingest.yaml => test.telemetry.yaml} | 4 +- .vscode/launch.json | 4 +- Common/Server/EnvironmentConfig.ts | 4 +- Common/ServiceRoute.ts | 4 +- Common/UI/Config.ts | 10 +-- .../StatusCheck.spec.ts | 14 ++-- .../Public/oneuptime/templates/_helpers.tpl | 12 ++-- .../templates/keda-scaledobjects.yaml | 10 +-- ...n-telemetry-ingest.yaml => telemetry.yaml} | 71 +++++++++---------- HelmChart/Public/oneuptime/values.schema.json | 2 +- HelmChart/Public/oneuptime/values.yaml | 2 +- .../otel-collector-config.template.yaml | 2 +- OpenTelemetryIngest/Config.ts | 50 ------------- .../.dockerignore | 0 .../.gitattributes | 0 {OpenTelemetryIngest => Telemetry}/.gitignore | 0 .../API/Fluent.ts | 0 .../API/Metrics.ts | 0 .../API/OTelIngest.ts | 0 .../API/Syslog.ts | 0 Telemetry/Config.ts | 49 +++++++++++++ .../Dockerfile.tpl | 8 +-- .../Docs/SysLog/README.md | 0 .../Docs/logData.example.json | 0 .../Docs/metricsData.example.json | 0 .../Docs/traceData.example.json | 0 {OpenTelemetryIngest => Telemetry}/Index.ts | 6 +- .../Jobs/TelemetryIngest/ProcessTelemetry.ts | 8 +-- .../Middleware/OtelRequestMiddleware.ts | 0 .../ProtoFiles/OTel/v1/common.proto | 0 .../ProtoFiles/OTel/v1/logs.proto | 0 .../ProtoFiles/OTel/v1/metrics.proto | 0 .../ProtoFiles/OTel/v1/resource.proto | 0 .../ProtoFiles/OTel/v1/traces.proto | 0 .../Services/FluentLogsIngestService.ts | 8 +-- .../Services/OtelIngestBaseService.ts | 0 .../Services/OtelLogsIngestService.ts | 8 +-- .../Services/OtelMetricsIngestService.ts | 8 +-- .../Services/OtelTracesIngestService.ts | 16 ++--- .../Services/Queue/FluentLogsQueueService.ts | 0 .../Services/Queue/LogsQueueService.ts | 0 .../Services/Queue/MetricsQueueService.ts | 0 .../Services/Queue/SyslogQueueService.ts | 0 .../Services/Queue/TelemetryQueueService.ts | 0 .../Services/Queue/TracesQueueService.ts | 0 .../Services/SyslogIngestService.ts | 8 +-- .../Tests/Utils/SyslogParser.test.ts | 0 .../Utils/Exception.ts | 0 .../Utils/SyslogParser.ts | 0 .../jest.config.json | 0 .../nodemon.json | 0 .../package-lock.json | 0 .../package.json | 0 .../tsconfig.json | 0 Tests/Scripts/status-check.sh | 2 +- config.example.env | 11 ++- docker-compose.base.yml | 12 ++-- docker-compose.dev.yml | 8 +-- docker-compose.yml | 6 +- 63 files changed, 204 insertions(+), 209 deletions(-) rename .github/workflows/{test.open-telemetry-ingest.yaml => test.telemetry.yaml} (77%) rename E2E/Tests/{OpenTelemetryIngest => TelemetryIngest}/StatusCheck.spec.ts (71%) rename HelmChart/Public/oneuptime/templates/{open-telemetry-ingest.yaml => telemetry.yaml} (56%) delete mode 100644 OpenTelemetryIngest/Config.ts rename {OpenTelemetryIngest => Telemetry}/.dockerignore (100%) rename {OpenTelemetryIngest => Telemetry}/.gitattributes (100%) rename {OpenTelemetryIngest => Telemetry}/.gitignore (100%) rename {OpenTelemetryIngest => Telemetry}/API/Fluent.ts (100%) rename {OpenTelemetryIngest => Telemetry}/API/Metrics.ts (100%) rename {OpenTelemetryIngest => Telemetry}/API/OTelIngest.ts (100%) rename {OpenTelemetryIngest => Telemetry}/API/Syslog.ts (100%) create mode 100644 Telemetry/Config.ts rename {OpenTelemetryIngest => Telemetry}/Dockerfile.tpl (92%) rename {OpenTelemetryIngest => Telemetry}/Docs/SysLog/README.md (100%) rename {OpenTelemetryIngest => Telemetry}/Docs/logData.example.json (100%) rename {OpenTelemetryIngest => Telemetry}/Docs/metricsData.example.json (100%) rename {OpenTelemetryIngest => Telemetry}/Docs/traceData.example.json (100%) rename {OpenTelemetryIngest => Telemetry}/Index.ts (92%) rename {OpenTelemetryIngest => Telemetry}/Jobs/TelemetryIngest/ProcessTelemetry.ts (94%) rename {OpenTelemetryIngest => Telemetry}/Middleware/OtelRequestMiddleware.ts (100%) rename {OpenTelemetryIngest => Telemetry}/ProtoFiles/OTel/v1/common.proto (100%) rename {OpenTelemetryIngest => Telemetry}/ProtoFiles/OTel/v1/logs.proto (100%) rename {OpenTelemetryIngest => Telemetry}/ProtoFiles/OTel/v1/metrics.proto (100%) rename {OpenTelemetryIngest => Telemetry}/ProtoFiles/OTel/v1/resource.proto (100%) rename {OpenTelemetryIngest => Telemetry}/ProtoFiles/OTel/v1/traces.proto (100%) rename {OpenTelemetryIngest => Telemetry}/Services/FluentLogsIngestService.ts (97%) rename {OpenTelemetryIngest => Telemetry}/Services/OtelIngestBaseService.ts (100%) rename {OpenTelemetryIngest => Telemetry}/Services/OtelLogsIngestService.ts (98%) rename {OpenTelemetryIngest => Telemetry}/Services/OtelMetricsIngestService.ts (98%) rename {OpenTelemetryIngest => Telemetry}/Services/OtelTracesIngestService.ts (98%) rename {OpenTelemetryIngest => Telemetry}/Services/Queue/FluentLogsQueueService.ts (100%) rename {OpenTelemetryIngest => Telemetry}/Services/Queue/LogsQueueService.ts (100%) rename {OpenTelemetryIngest => Telemetry}/Services/Queue/MetricsQueueService.ts (100%) rename {OpenTelemetryIngest => Telemetry}/Services/Queue/SyslogQueueService.ts (100%) rename {OpenTelemetryIngest => Telemetry}/Services/Queue/TelemetryQueueService.ts (100%) rename {OpenTelemetryIngest => Telemetry}/Services/Queue/TracesQueueService.ts (100%) rename {OpenTelemetryIngest => Telemetry}/Services/SyslogIngestService.ts (98%) rename {OpenTelemetryIngest => Telemetry}/Tests/Utils/SyslogParser.test.ts (100%) rename {OpenTelemetryIngest => Telemetry}/Utils/Exception.ts (100%) rename {OpenTelemetryIngest => Telemetry}/Utils/SyslogParser.ts (100%) rename {OpenTelemetryIngest => Telemetry}/jest.config.json (100%) rename {OpenTelemetryIngest => Telemetry}/nodemon.json (100%) rename {OpenTelemetryIngest => Telemetry}/package-lock.json (100%) rename {OpenTelemetryIngest => Telemetry}/package.json (100%) rename {OpenTelemetryIngest => Telemetry}/tsconfig.json (100%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3b549b7e1f..89b3e90ed4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -382,7 +382,7 @@ jobs: max_attempts: 3 command: sudo docker build --no-cache -f ./ServerMonitorIngest/Dockerfile . - docker-build-open-telemetry-ingest: + docker-build-telemetry: runs-on: ubuntu-latest env: CI_PIPELINE_ID: ${{github.run_number}} @@ -403,7 +403,7 @@ jobs: with: timeout_minutes: 45 max_attempts: 3 - command: sudo docker build --no-cache -f ./OpenTelemetryIngest/Dockerfile . + command: sudo docker build --no-cache -f ./Telemetry/Dockerfile . docker-build-incoming-request-ingest: runs-on: ubuntu-latest diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index 40027afb1e..5b7a0e84e4 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -319,7 +319,7 @@ jobs: max_attempts: 3 command: cd ServerMonitorIngest && npm install && npm run compile && npm run dep-check - compile-open-telemetry-ingest: + compile-telemetry: runs-on: ubuntu-latest env: CI_PIPELINE_ID: ${{github.run_number}} @@ -329,12 +329,12 @@ jobs: with: node-version: latest - run: cd Common && npm install - - name: Compile Open Telemetry Ingest + - name: Compile Telemetry uses: nick-fields/retry@v3 with: timeout_minutes: 30 max_attempts: 3 - command: cd OpenTelemetryIngest && npm install && npm run compile && npm run dep-check + command: cd Telemetry && npm install && npm run compile && npm run dep-check compile-incoming-request-ingest: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e7a6377dad..022a51ec13 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1108,7 +1108,7 @@ jobs: - open-telemetry-ingest-docker-image-deploy: + telemetry-docker-image-deploy: needs: [generate-build-number, read-version] runs-on: ubuntu-latest steps: @@ -1117,8 +1117,8 @@ jobs: uses: docker/metadata-action@v4 with: images: | - oneuptime/open-telemetry-ingest - ghcr.io/oneuptime/open-telemetry-ingest + oneuptime/telemetry + ghcr.io/oneuptime/telemetry tags: | type=raw,value=release,enable=true type=semver,value=${{needs.read-version.outputs.major_minor}}.${{needs.generate-build-number.outputs.build_number}},pattern={{version}},enable=true @@ -1144,7 +1144,7 @@ jobs: max_attempts: 3 command: npm run prerun - # Build and deploy open-telemetry-ingest. + # Build and deploy telemetry. - name: Login to Docker Hub uses: nick-fields/retry@v3 @@ -1171,22 +1171,22 @@ jobs: VERSION="${{needs.read-version.outputs.major_minor}}.${{needs.generate-build-number.outputs.build_number}}" docker buildx build \ --no-cache \ - --file ./OpenTelemetryIngest/Dockerfile \ + --file ./Telemetry/Dockerfile \ --platform linux/amd64,linux/arm64 \ --push \ - --tag oneuptime/open-telemetry-ingest:${VERSION} \ - --tag ghcr.io/oneuptime/open-telemetry-ingest:${VERSION} \ + --tag oneuptime/telemetry:${VERSION} \ + --tag ghcr.io/oneuptime/telemetry:${VERSION} \ --build-arg GIT_SHA=${{ github.sha }} \ --build-arg APP_VERSION=${VERSION} \ --build-arg IS_ENTERPRISE_EDITION=false \ . docker buildx build \ --no-cache \ - --file ./OpenTelemetryIngest/Dockerfile \ + --file ./Telemetry/Dockerfile \ --platform linux/amd64,linux/arm64 \ --push \ - --tag oneuptime/open-telemetry-ingest:enterprise-${VERSION} \ - --tag ghcr.io/oneuptime/open-telemetry-ingest:enterprise-${VERSION} \ + --tag oneuptime/telemetry:enterprise-${VERSION} \ + --tag ghcr.io/oneuptime/telemetry:enterprise-${VERSION} \ --build-arg GIT_SHA=${{ github.sha }} \ --build-arg APP_VERSION=${VERSION} \ --build-arg IS_ENTERPRISE_EDITION=true \ @@ -2340,7 +2340,7 @@ jobs: - test-docker-image-deploy - probe-ingest-docker-image-deploy - server-monitor-ingest-docker-image-deploy - - open-telemetry-ingest-docker-image-deploy + - telemetry-docker-image-deploy - incoming-request-ingest-docker-image-deploy - probe-docker-image-deploy - admin-dashboard-docker-image-deploy @@ -2371,7 +2371,7 @@ jobs: "test", "probe-ingest", "server-monitor-ingest", - "open-telemetry-ingest", + "telemetry", "incoming-request-ingest", "probe", "admin-dashboard", @@ -2438,7 +2438,7 @@ jobs: test-e2e-release-saas: runs-on: ubuntu-latest - needs: [open-telemetry-ingest-docker-image-deploy, publish-mcp-server, copilot-docker-image-deploy, docs-docker-image-deploy, api-reference-docker-image-deploy, workflow-docker-image-deploy, llm-docker-image-deploy, accounts-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, dashboard-docker-image-deploy, probe-ingest-docker-image-deploy, server-monitor-ingest-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, worker-docker-image-deploy, otel-collector-docker-image-deploy, probe-docker-image-deploy, status-page-docker-image-deploy, test-docker-image-deploy, test-server-docker-image-deploy, publish-npm-packages, e2e-docker-image-deploy, helm-chart-deploy, generate-build-number, read-version, nginx-docker-image-deploy, incoming-request-ingest-docker-image-deploy] + needs: [telemetry-docker-image-deploy, publish-mcp-server, copilot-docker-image-deploy, docs-docker-image-deploy, api-reference-docker-image-deploy, workflow-docker-image-deploy, llm-docker-image-deploy, accounts-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, dashboard-docker-image-deploy, probe-ingest-docker-image-deploy, server-monitor-ingest-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, worker-docker-image-deploy, otel-collector-docker-image-deploy, probe-docker-image-deploy, status-page-docker-image-deploy, test-docker-image-deploy, test-server-docker-image-deploy, publish-npm-packages, e2e-docker-image-deploy, helm-chart-deploy, generate-build-number, read-version, nginx-docker-image-deploy, incoming-request-ingest-docker-image-deploy] env: CI_PIPELINE_ID: ${{github.run_number}} steps: @@ -2525,7 +2525,7 @@ jobs: test-e2e-release-self-hosted: runs-on: ubuntu-latest # After all the jobs runs - needs: [open-telemetry-ingest-docker-image-deploy, publish-mcp-server, copilot-docker-image-deploy, incoming-request-ingest-docker-image-deploy, docs-docker-image-deploy, api-reference-docker-image-deploy, workflow-docker-image-deploy, llm-docker-image-deploy, accounts-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, dashboard-docker-image-deploy, probe-ingest-docker-image-deploy, server-monitor-ingest-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, worker-docker-image-deploy, otel-collector-docker-image-deploy, probe-docker-image-deploy, status-page-docker-image-deploy, test-docker-image-deploy, test-server-docker-image-deploy, publish-npm-packages, e2e-docker-image-deploy, helm-chart-deploy, generate-build-number, read-version, nginx-docker-image-deploy] + needs: [telemetry-docker-image-deploy, publish-mcp-server, copilot-docker-image-deploy, incoming-request-ingest-docker-image-deploy, docs-docker-image-deploy, api-reference-docker-image-deploy, workflow-docker-image-deploy, llm-docker-image-deploy, accounts-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, dashboard-docker-image-deploy, probe-ingest-docker-image-deploy, server-monitor-ingest-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, worker-docker-image-deploy, otel-collector-docker-image-deploy, probe-docker-image-deploy, status-page-docker-image-deploy, test-docker-image-deploy, test-server-docker-image-deploy, publish-npm-packages, e2e-docker-image-deploy, helm-chart-deploy, generate-build-number, read-version, nginx-docker-image-deploy] env: CI_PIPELINE_ID: ${{github.run_number}} steps: diff --git a/.github/workflows/test-release.yaml b/.github/workflows/test-release.yaml index 410d080f0e..85f8acdbca 100644 --- a/.github/workflows/test-release.yaml +++ b/.github/workflows/test-release.yaml @@ -1245,7 +1245,7 @@ on: --build-arg IS_ENTERPRISE_EDITION=true \ . - open-telemetry-ingest-docker-image-deploy: + telemetry-docker-image-deploy: needs: [read-version, generate-build-number] runs-on: ubuntu-latest steps: @@ -1254,8 +1254,8 @@ on: uses: docker/metadata-action@v4 with: images: | - oneuptime/open-telemetry-ingest - ghcr.io/oneuptime/open-telemetry-ingest + oneuptime/telemetry + ghcr.io/oneuptime/telemetry tags: | type=raw,value=test,enable=true type=raw,value=${{needs.read-version.outputs.major_minor}}.${{needs.generate-build-number.outputs.build_number}}-test,enable=true @@ -1309,26 +1309,26 @@ on: VERSION="${{needs.read-version.outputs.major_minor}}.${{needs.generate-build-number.outputs.build_number}}-test" docker buildx build \ --no-cache \ - --file ./OpenTelemetryIngest/Dockerfile \ + --file ./Telemetry/Dockerfile \ --platform linux/amd64,linux/arm64 \ --push \ - --tag oneuptime/open-telemetry-ingest:test \ - --tag oneuptime/open-telemetry-ingest:${VERSION} \ - --tag ghcr.io/oneuptime/open-telemetry-ingest:test \ - --tag ghcr.io/oneuptime/open-telemetry-ingest:${VERSION} \ + --tag oneuptime/telemetry:test \ + --tag oneuptime/telemetry:${VERSION} \ + --tag ghcr.io/oneuptime/telemetry:test \ + --tag ghcr.io/oneuptime/telemetry:${VERSION} \ --build-arg GIT_SHA=${{ github.sha }} \ --build-arg APP_VERSION=${VERSION} \ --build-arg IS_ENTERPRISE_EDITION=false \ . docker buildx build \ --no-cache \ - --file ./OpenTelemetryIngest/Dockerfile \ + --file ./Telemetry/Dockerfile \ --platform linux/amd64,linux/arm64 \ --push \ - --tag oneuptime/open-telemetry-ingest:enterprise-test \ - --tag oneuptime/open-telemetry-ingest:enterprise-${VERSION} \ - --tag ghcr.io/oneuptime/open-telemetry-ingest:enterprise-test \ - --tag ghcr.io/oneuptime/open-telemetry-ingest:enterprise-${VERSION} \ + --tag oneuptime/telemetry:enterprise-test \ + --tag oneuptime/telemetry:enterprise-${VERSION} \ + --tag ghcr.io/oneuptime/telemetry:enterprise-test \ + --tag ghcr.io/oneuptime/telemetry:enterprise-${VERSION} \ --build-arg GIT_SHA=${{ github.sha }} \ --build-arg APP_VERSION=${VERSION} \ --build-arg IS_ENTERPRISE_EDITION=true \ @@ -2236,7 +2236,7 @@ on: test-helm-chart: runs-on: ubuntu-latest - needs: [infrastructure-agent-deploy, publish-mcp-server, llm-docker-image-deploy, publish-terraform-provider, open-telemetry-ingest-docker-image-deploy, copilot-docker-image-deploy, docs-docker-image-deploy, worker-docker-image-deploy, workflow-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, api-reference-docker-image-deploy, test-server-docker-image-deploy, test-docker-image-deploy, probe-ingest-docker-image-deploy, server-monitor-ingest-docker-image-deploy, probe-docker-image-deploy, dashboard-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, accounts-docker-image-deploy, otel-collector-docker-image-deploy, status-page-docker-image-deploy, nginx-docker-image-deploy, e2e-docker-image-deploy, incoming-request-ingest-docker-image-deploy] + needs: [infrastructure-agent-deploy, publish-mcp-server, llm-docker-image-deploy, publish-terraform-provider, telemetry-docker-image-deploy, copilot-docker-image-deploy, docs-docker-image-deploy, worker-docker-image-deploy, workflow-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, api-reference-docker-image-deploy, test-server-docker-image-deploy, test-docker-image-deploy, probe-ingest-docker-image-deploy, server-monitor-ingest-docker-image-deploy, probe-docker-image-deploy, dashboard-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, accounts-docker-image-deploy, otel-collector-docker-image-deploy, status-page-docker-image-deploy, nginx-docker-image-deploy, e2e-docker-image-deploy, incoming-request-ingest-docker-image-deploy] env: CI_PIPELINE_ID: ${{github.run_number}} steps: diff --git a/.github/workflows/test.open-telemetry-ingest.yaml b/.github/workflows/test.telemetry.yaml similarity index 77% rename from .github/workflows/test.open-telemetry-ingest.yaml rename to .github/workflows/test.telemetry.yaml index 6c7eb9c165..1977c8d75e 100644 --- a/.github/workflows/test.open-telemetry-ingest.yaml +++ b/.github/workflows/test.telemetry.yaml @@ -1,4 +1,4 @@ -name: OpenTelemetryIngest Test +name: Telemetry Test on: pull_request: @@ -17,5 +17,5 @@ jobs: - uses: actions/setup-node@v4 with: node-version: latest - - run: cd OpenTelemetryIngest && npm install && npm run test + - run: cd Telemetry && npm install && npm run test diff --git a/.vscode/launch.json b/.vscode/launch.json index 0ee9afa4af..c8dec7e984 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -205,8 +205,8 @@ }, { "address": "127.0.0.1", - "localRoot": "${workspaceFolder}/OpenTelemetryIngest", - "name": "OpenTelemetryIngest: Debug with Docker", + "localRoot": "${workspaceFolder}/Telemetry", + "name": "Telemetry: Debug with Docker", "port": 9938, "remoteRoot": "/usr/src/app", "request": "attach", diff --git a/Common/Server/EnvironmentConfig.ts b/Common/Server/EnvironmentConfig.ts index d6d7af7fe3..5a1eb484f3 100644 --- a/Common/Server/EnvironmentConfig.ts +++ b/Common/Server/EnvironmentConfig.ts @@ -165,8 +165,8 @@ export const ProbeIngestHostname: Hostname = Hostname.fromString( ); export const OpenTelemetryIngestHostname: Hostname = Hostname.fromString( - `${process.env["SERVER_OPEN_TELEMETRY_INGEST_HOSTNAME"] || "localhost"}:${ - process.env["OPEN_TELEMETRY_INGEST_PORT"] || 80 + `${process.env["SERVER_TELEMETRY_HOSTNAME"] || "localhost"}:${ + process.env["TELEMETRY_PORT"] || 80 }`, ); diff --git a/Common/ServiceRoute.ts b/Common/ServiceRoute.ts index 6c497b9397..adc1f2e327 100644 --- a/Common/ServiceRoute.ts +++ b/Common/ServiceRoute.ts @@ -28,9 +28,7 @@ export const AdminDashboardRoute: Route = new Route("/admin"); export const ProbeIngestRoute: Route = new Route("/probe-ingest"); -export const OpenTelemetryIngestRoute: Route = new Route( - "/open-telemetry-ingest", -); +export const TelemetryRoute: Route = new Route("/telemetry"); export const IncomingRequestIngestRoute: Route = new Route( "/incoming-request-ingest", diff --git a/Common/UI/Config.ts b/Common/UI/Config.ts index 8553163752..0147f96054 100644 --- a/Common/UI/Config.ts +++ b/Common/UI/Config.ts @@ -16,7 +16,7 @@ import { StatusPageRoute, WorkflowRoute, IncomingRequestIngestRoute, - OpenTelemetryIngestRoute, + TelemetryRoute, } from "../ServiceRoute"; import Hostname from "../Types/API/Hostname"; import Protocol from "../Types/API/Protocol"; @@ -75,7 +75,7 @@ export const WORKFLOW_HOSTNAME: Hostname = Hostname.fromString(HOST); export const PROBE_INGEST_HOSTNAME: Hostname = Hostname.fromString(HOST); -export const OPEN_TELEMETRY_INGEST_HOSTNAME: Hostname = +export const TELEMETRY_HOSTNAME: Hostname = Hostname.fromString(HOST); export const INCOMING_REQUEST_INGEST_HOSTNAME: Hostname = @@ -115,10 +115,10 @@ export const STATUS_PAGE_API_URL: URL = new URL( new Route(StatusPageApiRoute.toString()), ); -export const OPEN_TELEMETRY_INGEST_URL: URL = new URL( +export const TELEMETRY_URL: URL = new URL( HTTP_PROTOCOL, - OPEN_TELEMETRY_INGEST_HOSTNAME, - new Route(OpenTelemetryIngestRoute.toString()), + TELEMETRY_HOSTNAME, + new Route(TelemetryRoute.toString()), ); export const IDENTITY_URL: URL = new URL( diff --git a/E2E/Tests/OpenTelemetryIngest/StatusCheck.spec.ts b/E2E/Tests/TelemetryIngest/StatusCheck.spec.ts similarity index 71% rename from E2E/Tests/OpenTelemetryIngest/StatusCheck.spec.ts rename to E2E/Tests/TelemetryIngest/StatusCheck.spec.ts index 7c6cb65689..7824776a2d 100644 --- a/E2E/Tests/OpenTelemetryIngest/StatusCheck.spec.ts +++ b/E2E/Tests/TelemetryIngest/StatusCheck.spec.ts @@ -2,8 +2,8 @@ import { BASE_URL } from "../../Config"; import { Page, expect, test } from "@playwright/test"; import URL from "Common/Types/API/URL"; -test.describe("check live and health check of the open-telemetry-ingest", () => { - test("check if open-telemetry-ingest status is ok", async ({ +test.describe("check live and health check of telemetry", () => { + test("check if telemetry status is ok", async ({ page, }: { page: Page; @@ -11,14 +11,14 @@ test.describe("check live and health check of the open-telemetry-ingest", () => page.setDefaultNavigationTimeout(120000); // 2 minutes await page.goto( `${URL.fromString(BASE_URL.toString()) - .addRoute("/open-telemetry-ingest/status") + .addRoute("/telemetry/status") .toString()}`, ); const content: string = await page.content(); expect(content).toContain('{"status":"ok"}'); }); - test("check if open-telemetry-ingest is ready", async ({ + test("check if telemetry is ready", async ({ page, }: { page: Page; @@ -26,14 +26,14 @@ test.describe("check live and health check of the open-telemetry-ingest", () => page.setDefaultNavigationTimeout(120000); // 2 minutes await page.goto( `${URL.fromString(BASE_URL.toString()) - .addRoute("/open-telemetry-ingest/status/ready") + .addRoute("/telemetry/status/ready") .toString()}`, ); const content: string = await page.content(); expect(content).toContain('{"status":"ok"}'); }); - test("check if open-telemetry-ingest is live", async ({ + test("check if telemetry is live", async ({ page, }: { page: Page; @@ -41,7 +41,7 @@ test.describe("check live and health check of the open-telemetry-ingest", () => page.setDefaultNavigationTimeout(120000); // 2 minutes await page.goto( `${URL.fromString(BASE_URL.toString()) - .addRoute("/open-telemetry-ingest/status/live") + .addRoute("/telemetry/status/live") .toString()}`, ); const content: string = await page.content(); diff --git a/HelmChart/Public/oneuptime/templates/_helpers.tpl b/HelmChart/Public/oneuptime/templates/_helpers.tpl index bd20b804e7..bc86a6437a 100644 --- a/HelmChart/Public/oneuptime/templates/_helpers.tpl +++ b/HelmChart/Public/oneuptime/templates/_helpers.tpl @@ -105,12 +105,12 @@ Usage: value: {{ $.Release.Name }}-app.{{ $.Release.Namespace }}.svc.{{ $.Values.global.clusterDomain }} - name: SERVER_PROBE_INGEST_HOSTNAME value: {{ $.Release.Name }}-probe-ingest.{{ $.Release.Namespace }}.svc.{{ $.Values.global.clusterDomain }} -- name: OPEN_TELEMETRY_INGEST_HOSTNAME - value: {{ $.Release.Name }}-open-telemetry-ingest.{{ $.Release.Namespace }}.svc.{{ $.Values.global.clusterDomain }} +- name: TELEMETRY_HOSTNAME + value: {{ $.Release.Name }}-telemetry.{{ $.Release.Namespace }}.svc.{{ $.Values.global.clusterDomain }} - name: SERVER_INCOMING_REQUEST_INGEST_HOSTNAME value: {{ $.Release.Name }}-incoming-request-ingest.{{ $.Release.Namespace }}.svc.{{ $.Values.global.clusterDomain }} -- name: SERVER_OPEN_TELEMETRY_INGEST_HOSTNAME - value: {{ $.Release.Name }}-open-telemetry-ingest.{{ $.Release.Namespace }}.svc.{{ $.Values.global.clusterDomain }} +- name: SERVER_TELEMETRY_HOSTNAME + value: {{ $.Release.Name }}-telemetry.{{ $.Release.Namespace }}.svc.{{ $.Values.global.clusterDomain }} - name: SERVER_TEST_SERVER_HOSTNAME value: {{ $.Release.Name }}-test-server.{{ $.Release.Namespace }}.svc.{{ $.Values.global.clusterDomain }} - name: SERVER_OTEL_COLLECTOR_HOSTNAME @@ -130,8 +130,8 @@ Usage: value: {{ $.Values.probeIngest.ports.http | squote }} - name: SERVER_MONITOR_INGEST_PORT value: {{ $.Values.serverMonitorIngest.ports.http | squote }} -- name: OPEN_TELEMETRY_INGEST_PORT - value: {{ $.Values.openTelemetryIngest.ports.http | squote }} +- name: TELEMETRY_PORT + value: {{ $.Values.telemetry.ports.http | squote }} - name: INCOMING_REQUEST_INGEST_PORT value: {{ $.Values.incomingRequestIngest.ports.http | squote }} - name: TEST_SERVER_PORT diff --git a/HelmChart/Public/oneuptime/templates/keda-scaledobjects.yaml b/HelmChart/Public/oneuptime/templates/keda-scaledobjects.yaml index 173d42c461..c24f3e5018 100644 --- a/HelmChart/Public/oneuptime/templates/keda-scaledobjects.yaml +++ b/HelmChart/Public/oneuptime/templates/keda-scaledobjects.yaml @@ -2,11 +2,11 @@ KEDA ScaledObjects for various services */}} -{{/* OpenTelemetry Ingest KEDA ScaledObject */}} -{{- if and .Values.keda.enabled .Values.openTelemetryIngest.keda.enabled (not .Values.openTelemetryIngest.disableAutoscaler) }} -{{- $metricsConfig := dict "enabled" .Values.openTelemetryIngest.keda.enabled "minReplicas" .Values.openTelemetryIngest.keda.minReplicas "maxReplicas" .Values.openTelemetryIngest.keda.maxReplicas "pollingInterval" .Values.openTelemetryIngest.keda.pollingInterval "cooldownPeriod" .Values.openTelemetryIngest.keda.cooldownPeriod "triggers" (list (dict "query" "oneuptime_telemetry_queue_size" "threshold" .Values.openTelemetryIngest.keda.queueSizeThreshold "port" .Values.openTelemetryIngest.ports.http)) }} -{{- $openTelemetryIngestKedaArgs := dict "ServiceName" "open-telemetry-ingest" "Release" .Release "Values" .Values "MetricsConfig" $metricsConfig "DisableAutoscaler" .Values.openTelemetryIngest.disableAutoscaler }} -{{- include "oneuptime.kedaScaledObject" $openTelemetryIngestKedaArgs }} +{{/* Telemetry KEDA ScaledObject */}} +{{- if and .Values.keda.enabled .Values.telemetry.keda.enabled (not .Values.telemetry.disableAutoscaler) }} +{{- $metricsConfig := dict "enabled" .Values.telemetry.keda.enabled "minReplicas" .Values.telemetry.keda.minReplicas "maxReplicas" .Values.telemetry.keda.maxReplicas "pollingInterval" .Values.telemetry.keda.pollingInterval "cooldownPeriod" .Values.telemetry.keda.cooldownPeriod "triggers" (list (dict "query" "oneuptime_telemetry_queue_size" "threshold" .Values.telemetry.keda.queueSizeThreshold "port" .Values.telemetry.ports.http)) }} +{{- $telemetryKedaArgs := dict "ServiceName" "telemetry" "Release" .Release "Values" .Values "MetricsConfig" $metricsConfig "DisableAutoscaler" .Values.telemetry.disableAutoscaler }} +{{- include "oneuptime.kedaScaledObject" $telemetryKedaArgs }} {{- end }} {{/* Incoming Request Ingest KEDA ScaledObject */}} diff --git a/HelmChart/Public/oneuptime/templates/open-telemetry-ingest.yaml b/HelmChart/Public/oneuptime/templates/telemetry.yaml similarity index 56% rename from HelmChart/Public/oneuptime/templates/open-telemetry-ingest.yaml rename to HelmChart/Public/oneuptime/templates/telemetry.yaml index b1660ee54f..86cae2899e 100644 --- a/HelmChart/Public/oneuptime/templates/open-telemetry-ingest.yaml +++ b/HelmChart/Public/oneuptime/templates/telemetry.yaml @@ -1,12 +1,12 @@ -# OneUptime open-telemetry-ingest Deployment +# OneUptime telemetry Deployment apiVersion: apps/v1 kind: Deployment metadata: - name: {{ printf "%s-%s" $.Release.Name "open-telemetry-ingest" }} + name: {{ printf "%s-%s" $.Release.Name "telemetry" }} namespace: {{ $.Release.Namespace }} labels: - app: {{ printf "%s-%s" $.Release.Name "open-telemetry-ingest" }} + app: {{ printf "%s-%s" $.Release.Name "telemetry" }} app.kubernetes.io/part-of: oneuptime app.kubernetes.io/managed-by: Helm appname: oneuptime @@ -16,11 +16,11 @@ metadata: spec: selector: matchLabels: - app: {{ printf "%s-%s" $.Release.Name "open-telemetry-ingest" }} - {{- if $.Values.openTelemetryIngest.replicaCount }} - replicas: {{ $.Values.openTelemetryIngest.replicaCount }} + app: {{ printf "%s-%s" $.Release.Name "telemetry" }} + {{- if $.Values.telemetry.replicaCount }} + replicas: {{ $.Values.telemetry.replicaCount }} {{- else }} - {{- if or (not $.Values.autoscaling.enabled) ($.Values.openTelemetryIngest.disableAutoscaler) }} + {{- if or (not $.Values.autoscaling.enabled) ($.Values.telemetry.disableAutoscaler) }} replicas: {{ $.Values.deployment.replicaCount }} {{- end }} {{- end }} @@ -28,7 +28,7 @@ spec: template: metadata: labels: - app: {{ printf "%s-%s" $.Release.Name "open-telemetry-ingest" }} + app: {{ printf "%s-%s" $.Release.Name "telemetry" }} {{- if $.Values.deployment.includeTimestampLabel }} date: "{{ now | unixEpoch }}" {{- end }} @@ -38,9 +38,9 @@ spec: - name: greenlockrc emptyDir: sizeLimit: "1Gi" - {{- if $.Values.openTelemetryIngest.podSecurityContext }} + {{- if $.Values.telemetry.podSecurityContext }} securityContext: - {{- toYaml $.Values.openTelemetryIngest.podSecurityContext | nindent 8 }} + {{- toYaml $.Values.telemetry.podSecurityContext | nindent 8 }} {{- else if $.Values.podSecurityContext }} securityContext: {{- toYaml $.Values.podSecurityContext | nindent 8 }} @@ -55,22 +55,22 @@ spec: {{- if $.Values.tolerations }} tolerations: {{- $.Values.tolerations | toYaml | nindent 8 }} {{- end }} - {{- if $.Values.openTelemetryIngest.nodeSelector }} + {{- if $.Values.telemetry.nodeSelector }} nodeSelector: - {{- toYaml $.Values.openTelemetryIngest.nodeSelector | nindent 8 }} + {{- toYaml $.Values.telemetry.nodeSelector | nindent 8 }} {{- else if $.Values.nodeSelector }} nodeSelector: {{- toYaml $.Values.nodeSelector | nindent 8 }} {{- end }} containers: - - image: {{ include "oneuptime.image" (dict "Values" $.Values "ServiceName" "open-telemetry-ingest") }} - name: {{ printf "%s-%s" $.Release.Name "open-telemetry-ingest" }} + - image: {{ include "oneuptime.image" (dict "Values" $.Values "ServiceName" "telemetry") }} + name: {{ printf "%s-%s" $.Release.Name "telemetry" }} {{- if $.Values.startupProbe.enabled }} # Startup probe startupProbe: httpGet: path: /status/live - port: {{ $.Values.openTelemetryIngest.ports.http }} + port: {{ $.Values.telemetry.ports.http }} periodSeconds: {{ $.Values.startupProbe.periodSeconds }} failureThreshold: {{ $.Values.startupProbe.failureThreshold }} {{- end }} @@ -79,7 +79,7 @@ spec: livenessProbe: httpGet: path: /status/live - port: {{ $.Values.openTelemetryIngest.ports.http }} + port: {{ $.Values.telemetry.ports.http }} periodSeconds: {{ $.Values.livenessProbe.periodSeconds }} timeoutSeconds: {{ $.Values.livenessProbe.timeoutSeconds }} initialDelaySeconds: {{ $.Values.livenessProbe.initialDelaySeconds }} @@ -89,14 +89,14 @@ spec: readinessProbe: httpGet: path: /status/ready - port: {{ $.Values.openTelemetryIngest.ports.http }} + port: {{ $.Values.telemetry.ports.http }} periodSeconds: {{ $.Values.readinessProbe.periodSeconds }} initialDelaySeconds: {{ $.Values.readinessProbe.initialDelaySeconds }} timeoutSeconds: {{ $.Values.readinessProbe.timeoutSeconds }} {{- end }} - {{- if $.Values.openTelemetryIngest.containerSecurityContext }} + {{- if $.Values.telemetry.containerSecurityContext }} securityContext: - {{- toYaml $.Values.openTelemetryIngest.containerSecurityContext | nindent 12 }} + {{- toYaml $.Values.telemetry.containerSecurityContext | nindent 12 }} {{- else if $.Values.containerSecurityContext }} securityContext: {{- toYaml $.Values.containerSecurityContext | nindent 12 }} @@ -106,32 +106,31 @@ spec: {{- include "oneuptime.env.common" . | nindent 12 }} {{- include "oneuptime.env.runtime" (dict "Values" $.Values "Release" $.Release) | nindent 12 }} - name: PORT - value: {{ $.Values.openTelemetryIngest.ports.http | quote }} + value: {{ $.Values.telemetry.ports.http | quote }} - name: DISABLE_TELEMETRY - value: {{ $.Values.openTelemetryIngest.disableTelemetryCollection | quote }} - - name: OPEN_TELEMETRY_INGEST_CONCURRENCY - value: {{ $.Values.openTelemetryIngest.concurrency | squote }} + value: {{ $.Values.telemetry.disableTelemetryCollection | quote }} + - name: TELEMETRY_CONCURRENCY + value: {{ $.Values.telemetry.concurrency | squote }} ports: - - containerPort: {{ $.Values.openTelemetryIngest.ports.http }} + - containerPort: {{ $.Values.telemetry.ports.http }} protocol: TCP name: http - {{- if $.Values.openTelemetryIngest.resources }} + {{- if $.Values.telemetry.resources }} resources: - {{- toYaml $.Values.openTelemetryIngest.resources | nindent 12 }} + {{- toYaml $.Values.telemetry.resources | nindent 12 }} {{- end }} restartPolicy: {{ $.Values.image.restartPolicy }} --- -# OneUptime open-telemetry-ingest Service -{{- $openTelemetryIngestPorts := dict "port" $.Values.openTelemetryIngest.ports.http -}} -{{- $openTelemetryIngestServiceArgs := dict "ServiceName" "open-telemetry-ingest" "Ports" $openTelemetryIngestPorts "Release" $.Release "Values" $.Values -}} -{{- include "oneuptime.service" $openTelemetryIngestServiceArgs }} +# OneUptime telemetry Service +{{- $telemetryPorts := dict "port" $.Values.telemetry.ports.http -}} +{{- $telemetryServiceArgs := dict "ServiceName" "telemetry" "Ports" $telemetryPorts "Release" $.Release "Values" $.Values -}} +{{- include "oneuptime.service" $telemetryServiceArgs }} --- -# OneUptime open-telemetry-ingest autoscaler -{{- if and (not $.Values.openTelemetryIngest.disableAutoscaler) (not (and $.Values.keda.enabled $.Values.openTelemetryIngest.keda.enabled)) }} -{{- $openTelemetryIngestAutoScalerArgs := dict "ServiceName" "open-telemetry-ingest" "Release" $.Release "Values" $.Values -}} -{{- include "oneuptime.autoscaler" $openTelemetryIngestAutoScalerArgs }} -{{- end }} ---- \ No newline at end of file +# OneUptime telemetry autoscaler +{{- if and (not $.Values.telemetry.disableAutoscaler) (not (and $.Values.keda.enabled $.Values.telemetry.keda.enabled)) }} +{{- $telemetryAutoScalerArgs := dict "ServiceName" "telemetry" "Release" $.Release "Values" $.Values -}} +{{- include "oneuptime.autoscaler" $telemetryAutoScalerArgs }} +{{- end }} \ No newline at end of file diff --git a/HelmChart/Public/oneuptime/values.schema.json b/HelmChart/Public/oneuptime/values.schema.json index a33a025bd9..a650e1a336 100644 --- a/HelmChart/Public/oneuptime/values.schema.json +++ b/HelmChart/Public/oneuptime/values.schema.json @@ -1735,7 +1735,7 @@ }, "additionalProperties": false }, - "openTelemetryIngest": { + "telemetry": { "type": "object", "properties": { "replicaCount": { diff --git a/HelmChart/Public/oneuptime/values.yaml b/HelmChart/Public/oneuptime/values.yaml index 62f145f469..08101a4f79 100644 --- a/HelmChart/Public/oneuptime/values.yaml +++ b/HelmChart/Public/oneuptime/values.yaml @@ -697,7 +697,7 @@ probeIngest: # Cooldown period after scaling (in seconds) cooldownPeriod: 300 -openTelemetryIngest: +telemetry: replicaCount: 1 disableTelemetryCollection: false disableAutoscaler: false diff --git a/OTelCollector/otel-collector-config.template.yaml b/OTelCollector/otel-collector-config.template.yaml index a4e0a45100..db7b7073ac 100644 --- a/OTelCollector/otel-collector-config.template.yaml +++ b/OTelCollector/otel-collector-config.template.yaml @@ -34,7 +34,7 @@ receivers: exporters: otlphttp: - endpoint: "http://{{ .Env.SERVER_OPEN_TELEMETRY_INGEST_HOSTNAME }}:{{ .Env.OPEN_TELEMETRY_INGEST_PORT }}/otlp" + endpoint: "http://{{ .Env.SERVER_TELEMETRY_HOSTNAME }}:{{ .Env.TELEMETRY_PORT }}/otlp" headers: {"Content-Type": "application/json"} auth: authenticator: headers_setter diff --git a/OpenTelemetryIngest/Config.ts b/OpenTelemetryIngest/Config.ts deleted file mode 100644 index 5a197ce306..0000000000 --- a/OpenTelemetryIngest/Config.ts +++ /dev/null @@ -1,50 +0,0 @@ -let concurrency: string | number = - process.env["OPEN_TELEMETRY_INGEST_CONCURRENCY"] || 100; - -if (typeof concurrency === "string") { - const parsed: number = parseInt(concurrency, 10); - concurrency = !isNaN(parsed) && parsed > 0 ? parsed : 100; -} - -export const OPEN_TELEMETRY_INGEST_CONCURRENCY: number = concurrency as number; - -type ParseBatchSizeFunction = (envKey: string, defaultValue: number) => number; - -const parseBatchSize: ParseBatchSizeFunction = ( - envKey: string, - defaultValue: number, -): number => { - const value: string | undefined = process.env[envKey]; - - if (!value) { - return defaultValue; - } - - const parsed: number = parseInt(value, 10); - - if (isNaN(parsed) || parsed <= 0) { - return defaultValue; - } - - return parsed; -}; - -export const OPEN_TELEMETRY_INGEST_LOG_FLUSH_BATCH_SIZE: number = - parseBatchSize("OPEN_TELEMETRY_INGEST_LOG_FLUSH_BATCH_SIZE", 1000); - -export const OPEN_TELEMETRY_INGEST_METRIC_FLUSH_BATCH_SIZE: number = - parseBatchSize("OPEN_TELEMETRY_INGEST_METRIC_FLUSH_BATCH_SIZE", 750); - -export const OPEN_TELEMETRY_INGEST_TRACE_FLUSH_BATCH_SIZE: number = - parseBatchSize("OPEN_TELEMETRY_INGEST_TRACE_FLUSH_BATCH_SIZE", 750); - -export const OPEN_TELEMETRY_INGEST_EXCEPTION_FLUSH_BATCH_SIZE: number = - parseBatchSize("OPEN_TELEMETRY_INGEST_EXCEPTION_FLUSH_BATCH_SIZE", 500); - -/* - * Some telemetry batches can be large and take >30s (BullMQ default lock) to process. - * Allow configuring a longer lock duration (in ms) to avoid premature stall detection. - */ - -// 10 minutes. -export const OPEN_TELEMETRY_INGEST_LOCK_DURATION_MS: number = 10 * 60 * 1000; diff --git a/OpenTelemetryIngest/.dockerignore b/Telemetry/.dockerignore similarity index 100% rename from OpenTelemetryIngest/.dockerignore rename to Telemetry/.dockerignore diff --git a/OpenTelemetryIngest/.gitattributes b/Telemetry/.gitattributes similarity index 100% rename from OpenTelemetryIngest/.gitattributes rename to Telemetry/.gitattributes diff --git a/OpenTelemetryIngest/.gitignore b/Telemetry/.gitignore similarity index 100% rename from OpenTelemetryIngest/.gitignore rename to Telemetry/.gitignore diff --git a/OpenTelemetryIngest/API/Fluent.ts b/Telemetry/API/Fluent.ts similarity index 100% rename from OpenTelemetryIngest/API/Fluent.ts rename to Telemetry/API/Fluent.ts diff --git a/OpenTelemetryIngest/API/Metrics.ts b/Telemetry/API/Metrics.ts similarity index 100% rename from OpenTelemetryIngest/API/Metrics.ts rename to Telemetry/API/Metrics.ts diff --git a/OpenTelemetryIngest/API/OTelIngest.ts b/Telemetry/API/OTelIngest.ts similarity index 100% rename from OpenTelemetryIngest/API/OTelIngest.ts rename to Telemetry/API/OTelIngest.ts diff --git a/OpenTelemetryIngest/API/Syslog.ts b/Telemetry/API/Syslog.ts similarity index 100% rename from OpenTelemetryIngest/API/Syslog.ts rename to Telemetry/API/Syslog.ts diff --git a/Telemetry/Config.ts b/Telemetry/Config.ts new file mode 100644 index 0000000000..4082470ba5 --- /dev/null +++ b/Telemetry/Config.ts @@ -0,0 +1,49 @@ +let concurrency: string | number = process.env["TELEMETRY_CONCURRENCY"] || 100; + +if (typeof concurrency === "string") { + const parsed: number = parseInt(concurrency, 10); + concurrency = !isNaN(parsed) && parsed > 0 ? parsed : 100; +} + +export const TELEMETRY_CONCURRENCY: number = concurrency as number; + +type ParseBatchSizeFunction = (envKey: string, defaultValue: number) => number; + +const parseBatchSize: ParseBatchSizeFunction = ( + envKey: string, + defaultValue: number, +): number => { + const value: string | undefined = process.env[envKey]; + + if (!value) { + return defaultValue; + } + + const parsed: number = parseInt(value, 10); + + if (isNaN(parsed) || parsed <= 0) { + return defaultValue; + } + + return parsed; +}; + +export const TELEMETRY_LOG_FLUSH_BATCH_SIZE: number = + parseBatchSize("TELEMETRY_LOG_FLUSH_BATCH_SIZE", 1000); + +export const TELEMETRY_METRIC_FLUSH_BATCH_SIZE: number = + parseBatchSize("TELEMETRY_METRIC_FLUSH_BATCH_SIZE", 750); + +export const TELEMETRY_TRACE_FLUSH_BATCH_SIZE: number = + parseBatchSize("TELEMETRY_TRACE_FLUSH_BATCH_SIZE", 750); + +export const TELEMETRY_EXCEPTION_FLUSH_BATCH_SIZE: number = + parseBatchSize("TELEMETRY_EXCEPTION_FLUSH_BATCH_SIZE", 500); + +/* + * Some telemetry batches can be large and take >30s (BullMQ default lock) to process. + * Allow configuring a longer lock duration (in ms) to avoid premature stall detection. + */ + +// 10 minutes. +export const TELEMETRY_LOCK_DURATION_MS: number = 10 * 60 * 1000; diff --git a/OpenTelemetryIngest/Dockerfile.tpl b/Telemetry/Dockerfile.tpl similarity index 92% rename from OpenTelemetryIngest/Dockerfile.tpl rename to Telemetry/Dockerfile.tpl index 20a4d035e5..471d25c3bc 100644 --- a/OpenTelemetryIngest/Dockerfile.tpl +++ b/Telemetry/Dockerfile.tpl @@ -1,5 +1,5 @@ # -# OneUptime-OpenTelemetryIngest Dockerfile +# OneUptime-Telemetry Dockerfile # # Pull base image nodejs image. @@ -65,11 +65,11 @@ WORKDIR /usr/src/app RUN PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=0 npx playwright install --with-deps # Install app dependencies -COPY ./OpenTelemetryIngest/package*.json /usr/src/app/ +COPY ./Telemetry/package*.json /usr/src/app/ RUN npm install # Expose ports. -# - 3403: OneUptime-OpenTelemetryIngest +# - 3403: OneUptime-Telemetry EXPOSE 3403 {{ if eq .Env.ENVIRONMENT "development" }} @@ -77,7 +77,7 @@ EXPOSE 3403 CMD [ "npm", "run", "dev" ] {{ else }} # Copy app source -COPY ./OpenTelemetryIngest /usr/src/app +COPY ./Telemetry /usr/src/app # Bundle app source RUN npm run compile # Set permission to write logs and cache in case container run as non root diff --git a/OpenTelemetryIngest/Docs/SysLog/README.md b/Telemetry/Docs/SysLog/README.md similarity index 100% rename from OpenTelemetryIngest/Docs/SysLog/README.md rename to Telemetry/Docs/SysLog/README.md diff --git a/OpenTelemetryIngest/Docs/logData.example.json b/Telemetry/Docs/logData.example.json similarity index 100% rename from OpenTelemetryIngest/Docs/logData.example.json rename to Telemetry/Docs/logData.example.json diff --git a/OpenTelemetryIngest/Docs/metricsData.example.json b/Telemetry/Docs/metricsData.example.json similarity index 100% rename from OpenTelemetryIngest/Docs/metricsData.example.json rename to Telemetry/Docs/metricsData.example.json diff --git a/OpenTelemetryIngest/Docs/traceData.example.json b/Telemetry/Docs/traceData.example.json similarity index 100% rename from OpenTelemetryIngest/Docs/traceData.example.json rename to Telemetry/Docs/traceData.example.json diff --git a/OpenTelemetryIngest/Index.ts b/Telemetry/Index.ts similarity index 92% rename from OpenTelemetryIngest/Index.ts rename to Telemetry/Index.ts index da4b7989af..842b960f4a 100644 --- a/OpenTelemetryIngest/Index.ts +++ b/Telemetry/Index.ts @@ -13,13 +13,13 @@ import Realtime from "Common/Server/Utils/Realtime"; import App from "Common/Server/Utils/StartServer"; import Telemetry from "Common/Server/Utils/Telemetry"; import "./Jobs/TelemetryIngest/ProcessTelemetry"; -import { OPEN_TELEMETRY_INGEST_CONCURRENCY } from "./Config"; +import { TELEMETRY_CONCURRENCY } from "./Config"; import type { StatusAPIOptions } from "Common/Server/API/StatusAPI"; import "ejs"; const app: ExpressApplication = Express.getExpressApp(); -const APP_NAME: string = "open-telemetry-ingest"; +const APP_NAME: string = "telemetry"; const ROUTE_PREFIXES: Array = [`/${APP_NAME}`, "/"]; app.use(ROUTE_PREFIXES, OTelIngestAPI); @@ -44,7 +44,7 @@ const init: PromiseVoidFunction = async (): Promise => { }); logger.info( - `OpenTelemetryIngest Service - Queue concurrency: ${OPEN_TELEMETRY_INGEST_CONCURRENCY}`, + `Telemetry Service - Queue concurrency: ${TELEMETRY_CONCURRENCY}`, ); // init the app diff --git a/OpenTelemetryIngest/Jobs/TelemetryIngest/ProcessTelemetry.ts b/Telemetry/Jobs/TelemetryIngest/ProcessTelemetry.ts similarity index 94% rename from OpenTelemetryIngest/Jobs/TelemetryIngest/ProcessTelemetry.ts rename to Telemetry/Jobs/TelemetryIngest/ProcessTelemetry.ts index 918134cbf4..863f37803c 100644 --- a/OpenTelemetryIngest/Jobs/TelemetryIngest/ProcessTelemetry.ts +++ b/Telemetry/Jobs/TelemetryIngest/ProcessTelemetry.ts @@ -13,8 +13,8 @@ import { QueueJob, QueueName } from "Common/Server/Infrastructure/Queue"; import QueueWorker from "Common/Server/Infrastructure/QueueWorker"; import ObjectID from "Common/Types/ObjectID"; import { - OPEN_TELEMETRY_INGEST_CONCURRENCY, - OPEN_TELEMETRY_INGEST_LOCK_DURATION_MS, + TELEMETRY_CONCURRENCY, + TELEMETRY_LOCK_DURATION_MS, } from "../../Config"; // Set up the unified worker for processing telemetry queue @@ -81,8 +81,8 @@ QueueWorker.getWorker( } }, { - concurrency: OPEN_TELEMETRY_INGEST_CONCURRENCY, - lockDuration: OPEN_TELEMETRY_INGEST_LOCK_DURATION_MS, + concurrency: TELEMETRY_CONCURRENCY, + lockDuration: TELEMETRY_LOCK_DURATION_MS, // allow a couple of stall recoveries before marking failed if genuinely stuck maxStalledCount: 2, }, diff --git a/OpenTelemetryIngest/Middleware/OtelRequestMiddleware.ts b/Telemetry/Middleware/OtelRequestMiddleware.ts similarity index 100% rename from OpenTelemetryIngest/Middleware/OtelRequestMiddleware.ts rename to Telemetry/Middleware/OtelRequestMiddleware.ts diff --git a/OpenTelemetryIngest/ProtoFiles/OTel/v1/common.proto b/Telemetry/ProtoFiles/OTel/v1/common.proto similarity index 100% rename from OpenTelemetryIngest/ProtoFiles/OTel/v1/common.proto rename to Telemetry/ProtoFiles/OTel/v1/common.proto diff --git a/OpenTelemetryIngest/ProtoFiles/OTel/v1/logs.proto b/Telemetry/ProtoFiles/OTel/v1/logs.proto similarity index 100% rename from OpenTelemetryIngest/ProtoFiles/OTel/v1/logs.proto rename to Telemetry/ProtoFiles/OTel/v1/logs.proto diff --git a/OpenTelemetryIngest/ProtoFiles/OTel/v1/metrics.proto b/Telemetry/ProtoFiles/OTel/v1/metrics.proto similarity index 100% rename from OpenTelemetryIngest/ProtoFiles/OTel/v1/metrics.proto rename to Telemetry/ProtoFiles/OTel/v1/metrics.proto diff --git a/OpenTelemetryIngest/ProtoFiles/OTel/v1/resource.proto b/Telemetry/ProtoFiles/OTel/v1/resource.proto similarity index 100% rename from OpenTelemetryIngest/ProtoFiles/OTel/v1/resource.proto rename to Telemetry/ProtoFiles/OTel/v1/resource.proto diff --git a/OpenTelemetryIngest/ProtoFiles/OTel/v1/traces.proto b/Telemetry/ProtoFiles/OTel/v1/traces.proto similarity index 100% rename from OpenTelemetryIngest/ProtoFiles/OTel/v1/traces.proto rename to Telemetry/ProtoFiles/OTel/v1/traces.proto diff --git a/OpenTelemetryIngest/Services/FluentLogsIngestService.ts b/Telemetry/Services/FluentLogsIngestService.ts similarity index 97% rename from OpenTelemetryIngest/Services/FluentLogsIngestService.ts rename to Telemetry/Services/FluentLogsIngestService.ts index 8ead5131d9..2bc0d3f3d3 100644 --- a/OpenTelemetryIngest/Services/FluentLogsIngestService.ts +++ b/Telemetry/Services/FluentLogsIngestService.ts @@ -22,7 +22,7 @@ import OTelIngestService, { import LogService from "Common/Server/Services/LogService"; import OtelIngestBaseService from "./OtelIngestBaseService"; import FluentLogsQueueService from "./Queue/FluentLogsQueueService"; -import { OPEN_TELEMETRY_INGEST_LOG_FLUSH_BATCH_SIZE } from "../Config"; +import { TELEMETRY_LOG_FLUSH_BATCH_SIZE } from "../Config"; export default class FluentLogsIngestService extends OtelIngestBaseService { private static readonly DEFAULT_SERVICE_NAME: string = "Fluentd"; @@ -150,7 +150,7 @@ export default class FluentLogsIngestService extends OtelIngestBaseService { dbLogs.push(logRow); processed++; - if (dbLogs.length >= OPEN_TELEMETRY_INGEST_LOG_FLUSH_BATCH_SIZE) { + if (dbLogs.length >= TELEMETRY_LOG_FLUSH_BATCH_SIZE) { await this.flushLogsBuffer(dbLogs); } } catch (processingError) { @@ -302,12 +302,12 @@ export default class FluentLogsIngestService extends OtelIngestBaseService { force: boolean = false, ): Promise { while ( - logs.length >= OPEN_TELEMETRY_INGEST_LOG_FLUSH_BATCH_SIZE || + logs.length >= TELEMETRY_LOG_FLUSH_BATCH_SIZE || (force && logs.length > 0) ) { const batchSize: number = Math.min( logs.length, - OPEN_TELEMETRY_INGEST_LOG_FLUSH_BATCH_SIZE, + TELEMETRY_LOG_FLUSH_BATCH_SIZE, ); const batch: Array = logs.splice(0, batchSize); diff --git a/OpenTelemetryIngest/Services/OtelIngestBaseService.ts b/Telemetry/Services/OtelIngestBaseService.ts similarity index 100% rename from OpenTelemetryIngest/Services/OtelIngestBaseService.ts rename to Telemetry/Services/OtelIngestBaseService.ts diff --git a/OpenTelemetryIngest/Services/OtelLogsIngestService.ts b/Telemetry/Services/OtelLogsIngestService.ts similarity index 98% rename from OpenTelemetryIngest/Services/OtelLogsIngestService.ts rename to Telemetry/Services/OtelLogsIngestService.ts index 12ade43099..8d5f594eb1 100644 --- a/OpenTelemetryIngest/Services/OtelLogsIngestService.ts +++ b/Telemetry/Services/OtelLogsIngestService.ts @@ -22,7 +22,7 @@ import logger from "Common/Server/Utils/Logger"; import CaptureSpan from "Common/Server/Utils/Telemetry/CaptureSpan"; import LogsQueueService from "./Queue/LogsQueueService"; import OtelIngestBaseService from "./OtelIngestBaseService"; -import { OPEN_TELEMETRY_INGEST_LOG_FLUSH_BATCH_SIZE } from "../Config"; +import { TELEMETRY_LOG_FLUSH_BATCH_SIZE } from "../Config"; import LogService from "Common/Server/Services/LogService"; export default class OtelLogsIngestService extends OtelIngestBaseService { @@ -31,12 +31,12 @@ export default class OtelLogsIngestService extends OtelIngestBaseService { force: boolean = false, ): Promise { while ( - logs.length >= OPEN_TELEMETRY_INGEST_LOG_FLUSH_BATCH_SIZE || + logs.length >= TELEMETRY_LOG_FLUSH_BATCH_SIZE || (force && logs.length > 0) ) { const batchSize: number = Math.min( logs.length, - OPEN_TELEMETRY_INGEST_LOG_FLUSH_BATCH_SIZE, + TELEMETRY_LOG_FLUSH_BATCH_SIZE, ); const batch: Array = logs.splice(0, batchSize); @@ -307,7 +307,7 @@ export default class OtelLogsIngestService extends OtelIngestBaseService { totalLogsProcessed++; if ( - dbLogs.length >= OPEN_TELEMETRY_INGEST_LOG_FLUSH_BATCH_SIZE + dbLogs.length >= TELEMETRY_LOG_FLUSH_BATCH_SIZE ) { await this.flushLogsBuffer(dbLogs); } diff --git a/OpenTelemetryIngest/Services/OtelMetricsIngestService.ts b/Telemetry/Services/OtelMetricsIngestService.ts similarity index 98% rename from OpenTelemetryIngest/Services/OtelMetricsIngestService.ts rename to Telemetry/Services/OtelMetricsIngestService.ts index 434800920d..6065ec330d 100644 --- a/OpenTelemetryIngest/Services/OtelMetricsIngestService.ts +++ b/Telemetry/Services/OtelMetricsIngestService.ts @@ -26,7 +26,7 @@ import MetricType from "Common/Models/DatabaseModels/MetricType"; import TelemetryService from "Common/Models/DatabaseModels/TelemetryService"; import MetricsQueueService from "./Queue/MetricsQueueService"; import OtelIngestBaseService from "./OtelIngestBaseService"; -import { OPEN_TELEMETRY_INGEST_METRIC_FLUSH_BATCH_SIZE } from "../Config"; +import { TELEMETRY_METRIC_FLUSH_BATCH_SIZE } from "../Config"; import OneUptimeDate from "Common/Types/Date"; import MetricService from "Common/Server/Services/MetricService"; @@ -43,12 +43,12 @@ export default class OtelMetricsIngestService extends OtelIngestBaseService { force: boolean = false, ): Promise { while ( - metrics.length >= OPEN_TELEMETRY_INGEST_METRIC_FLUSH_BATCH_SIZE || + metrics.length >= TELEMETRY_METRIC_FLUSH_BATCH_SIZE || (force && metrics.length > 0) ) { const batchSize: number = Math.min( metrics.length, - OPEN_TELEMETRY_INGEST_METRIC_FLUSH_BATCH_SIZE, + TELEMETRY_METRIC_FLUSH_BATCH_SIZE, ); const batch: Array = metrics.splice(0, batchSize); @@ -313,7 +313,7 @@ export default class OtelMetricsIngestService extends OtelIngestBaseService { if ( dbMetrics.length >= - OPEN_TELEMETRY_INGEST_METRIC_FLUSH_BATCH_SIZE + TELEMETRY_METRIC_FLUSH_BATCH_SIZE ) { await this.flushMetricsBuffer(dbMetrics); } diff --git a/OpenTelemetryIngest/Services/OtelTracesIngestService.ts b/Telemetry/Services/OtelTracesIngestService.ts similarity index 98% rename from OpenTelemetryIngest/Services/OtelTracesIngestService.ts rename to Telemetry/Services/OtelTracesIngestService.ts index 7d5c5dd0b6..7692b47b67 100644 --- a/OpenTelemetryIngest/Services/OtelTracesIngestService.ts +++ b/Telemetry/Services/OtelTracesIngestService.ts @@ -30,8 +30,8 @@ import Text from "Common/Types/Text"; import TracesQueueService from "./Queue/TracesQueueService"; import OtelIngestBaseService from "./OtelIngestBaseService"; import { - OPEN_TELEMETRY_INGEST_EXCEPTION_FLUSH_BATCH_SIZE, - OPEN_TELEMETRY_INGEST_TRACE_FLUSH_BATCH_SIZE, + TELEMETRY_EXCEPTION_FLUSH_BATCH_SIZE, + TELEMETRY_TRACE_FLUSH_BATCH_SIZE, } from "../Config"; type ParsedUnixNano = { @@ -63,12 +63,12 @@ export default class OtelTracesIngestService extends OtelIngestBaseService { force: boolean = false, ): Promise { while ( - spans.length >= OPEN_TELEMETRY_INGEST_TRACE_FLUSH_BATCH_SIZE || + spans.length >= TELEMETRY_TRACE_FLUSH_BATCH_SIZE || (force && spans.length > 0) ) { const batchSize: number = Math.min( spans.length, - OPEN_TELEMETRY_INGEST_TRACE_FLUSH_BATCH_SIZE, + TELEMETRY_TRACE_FLUSH_BATCH_SIZE, ); const batch: Array = spans.splice(0, batchSize); @@ -85,12 +85,12 @@ export default class OtelTracesIngestService extends OtelIngestBaseService { force: boolean = false, ): Promise { while ( - exceptions.length >= OPEN_TELEMETRY_INGEST_EXCEPTION_FLUSH_BATCH_SIZE || + exceptions.length >= TELEMETRY_EXCEPTION_FLUSH_BATCH_SIZE || (force && exceptions.length > 0) ) { const batchSize: number = Math.min( exceptions.length, - OPEN_TELEMETRY_INGEST_EXCEPTION_FLUSH_BATCH_SIZE, + TELEMETRY_EXCEPTION_FLUSH_BATCH_SIZE, ); const batch: Array = exceptions.splice(0, batchSize); @@ -363,14 +363,14 @@ export default class OtelTracesIngestService extends OtelIngestBaseService { if ( dbSpans.length >= - OPEN_TELEMETRY_INGEST_TRACE_FLUSH_BATCH_SIZE + TELEMETRY_TRACE_FLUSH_BATCH_SIZE ) { await this.flushSpansBuffer(dbSpans); } if ( dbExceptions.length >= - OPEN_TELEMETRY_INGEST_EXCEPTION_FLUSH_BATCH_SIZE + TELEMETRY_EXCEPTION_FLUSH_BATCH_SIZE ) { await this.flushExceptionsBuffer(dbExceptions); } diff --git a/OpenTelemetryIngest/Services/Queue/FluentLogsQueueService.ts b/Telemetry/Services/Queue/FluentLogsQueueService.ts similarity index 100% rename from OpenTelemetryIngest/Services/Queue/FluentLogsQueueService.ts rename to Telemetry/Services/Queue/FluentLogsQueueService.ts diff --git a/OpenTelemetryIngest/Services/Queue/LogsQueueService.ts b/Telemetry/Services/Queue/LogsQueueService.ts similarity index 100% rename from OpenTelemetryIngest/Services/Queue/LogsQueueService.ts rename to Telemetry/Services/Queue/LogsQueueService.ts diff --git a/OpenTelemetryIngest/Services/Queue/MetricsQueueService.ts b/Telemetry/Services/Queue/MetricsQueueService.ts similarity index 100% rename from OpenTelemetryIngest/Services/Queue/MetricsQueueService.ts rename to Telemetry/Services/Queue/MetricsQueueService.ts diff --git a/OpenTelemetryIngest/Services/Queue/SyslogQueueService.ts b/Telemetry/Services/Queue/SyslogQueueService.ts similarity index 100% rename from OpenTelemetryIngest/Services/Queue/SyslogQueueService.ts rename to Telemetry/Services/Queue/SyslogQueueService.ts diff --git a/OpenTelemetryIngest/Services/Queue/TelemetryQueueService.ts b/Telemetry/Services/Queue/TelemetryQueueService.ts similarity index 100% rename from OpenTelemetryIngest/Services/Queue/TelemetryQueueService.ts rename to Telemetry/Services/Queue/TelemetryQueueService.ts diff --git a/OpenTelemetryIngest/Services/Queue/TracesQueueService.ts b/Telemetry/Services/Queue/TracesQueueService.ts similarity index 100% rename from OpenTelemetryIngest/Services/Queue/TracesQueueService.ts rename to Telemetry/Services/Queue/TracesQueueService.ts diff --git a/OpenTelemetryIngest/Services/SyslogIngestService.ts b/Telemetry/Services/SyslogIngestService.ts similarity index 98% rename from OpenTelemetryIngest/Services/SyslogIngestService.ts rename to Telemetry/Services/SyslogIngestService.ts index adfb2e0997..379f464f61 100644 --- a/OpenTelemetryIngest/Services/SyslogIngestService.ts +++ b/Telemetry/Services/SyslogIngestService.ts @@ -22,7 +22,7 @@ import LogService from "Common/Server/Services/LogService"; import logger from "Common/Server/Utils/Logger"; import OtelIngestBaseService from "./OtelIngestBaseService"; import SyslogQueueService from "./Queue/SyslogQueueService"; -import { OPEN_TELEMETRY_INGEST_LOG_FLUSH_BATCH_SIZE } from "../Config"; +import { TELEMETRY_LOG_FLUSH_BATCH_SIZE } from "../Config"; import { ParsedSyslogMessage, ParsedSyslogStructuredData, @@ -216,7 +216,7 @@ export default class SyslogIngestService extends OtelIngestBaseService { dbLogs.push(logRow); processed++; - if (dbLogs.length >= OPEN_TELEMETRY_INGEST_LOG_FLUSH_BATCH_SIZE) { + if (dbLogs.length >= TELEMETRY_LOG_FLUSH_BATCH_SIZE) { await this.flushLogsBuffer(dbLogs); } } catch (processingError) { @@ -394,12 +394,12 @@ export default class SyslogIngestService extends OtelIngestBaseService { force: boolean = false, ): Promise { while ( - logs.length >= OPEN_TELEMETRY_INGEST_LOG_FLUSH_BATCH_SIZE || + logs.length >= TELEMETRY_LOG_FLUSH_BATCH_SIZE || (force && logs.length > 0) ) { const batchSize: number = Math.min( logs.length, - OPEN_TELEMETRY_INGEST_LOG_FLUSH_BATCH_SIZE, + TELEMETRY_LOG_FLUSH_BATCH_SIZE, ); const batch: Array = logs.splice(0, batchSize); diff --git a/OpenTelemetryIngest/Tests/Utils/SyslogParser.test.ts b/Telemetry/Tests/Utils/SyslogParser.test.ts similarity index 100% rename from OpenTelemetryIngest/Tests/Utils/SyslogParser.test.ts rename to Telemetry/Tests/Utils/SyslogParser.test.ts diff --git a/OpenTelemetryIngest/Utils/Exception.ts b/Telemetry/Utils/Exception.ts similarity index 100% rename from OpenTelemetryIngest/Utils/Exception.ts rename to Telemetry/Utils/Exception.ts diff --git a/OpenTelemetryIngest/Utils/SyslogParser.ts b/Telemetry/Utils/SyslogParser.ts similarity index 100% rename from OpenTelemetryIngest/Utils/SyslogParser.ts rename to Telemetry/Utils/SyslogParser.ts diff --git a/OpenTelemetryIngest/jest.config.json b/Telemetry/jest.config.json similarity index 100% rename from OpenTelemetryIngest/jest.config.json rename to Telemetry/jest.config.json diff --git a/OpenTelemetryIngest/nodemon.json b/Telemetry/nodemon.json similarity index 100% rename from OpenTelemetryIngest/nodemon.json rename to Telemetry/nodemon.json diff --git a/OpenTelemetryIngest/package-lock.json b/Telemetry/package-lock.json similarity index 100% rename from OpenTelemetryIngest/package-lock.json rename to Telemetry/package-lock.json diff --git a/OpenTelemetryIngest/package.json b/Telemetry/package.json similarity index 100% rename from OpenTelemetryIngest/package.json rename to Telemetry/package.json diff --git a/OpenTelemetryIngest/tsconfig.json b/Telemetry/tsconfig.json similarity index 100% rename from OpenTelemetryIngest/tsconfig.json rename to Telemetry/tsconfig.json diff --git a/Tests/Scripts/status-check.sh b/Tests/Scripts/status-check.sh index 4196ac7cb2..021265e35b 100644 --- a/Tests/Scripts/status-check.sh +++ b/Tests/Scripts/status-check.sh @@ -49,7 +49,7 @@ bash $scriptDir/endpoint-status.sh "Admin Dashboard (Ready Check)" $HOST_TO_CHEC bash $scriptDir/endpoint-status.sh "ProbeIngest (Ready Check)" $HOST_TO_CHECK/probe-ingest/status/ready -bash $scriptDir/endpoint-status.sh "OpenTelemetry Ingest (Ready Check)" $HOST_TO_CHECK/open-telemetry-ingest/status/ready +bash $scriptDir/endpoint-status.sh "Telemetry (Ready Check)" $HOST_TO_CHECK/telemetry/status/ready bash $scriptDir/endpoint-status.sh "ProbeIngest (Status Check)" $HOST_TO_CHECK/probe-ingest/status diff --git a/config.example.env b/config.example.env index c6dd1b5707..2e74c57034 100644 --- a/config.example.env +++ b/config.example.env @@ -92,15 +92,14 @@ REDIS_TLS_SENTINEL_MODE=false # Hostnames. Usually does not need to change. PROBE_INGEST_HOSTNAME=probe-ingest:3400 -FLUENT_LOGS_HOSTNAME=open-telemetry-ingest:3403 INCOMING_REQUEST_INGEST_HOSTNAME=incoming-request-ingest:3402 -OPEN_TELEMETRY_INGEST_HOSTNAME=otel-telemetry-ingest:3403 +TELEMETRY_HOSTNAME=telemetry:3403 SERVER_ACCOUNTS_HOSTNAME=accounts SERVER_APP_HOSTNAME=app SERVER_PROBE_INGEST_HOSTNAME=probe-ingest SERVER_SERVER_MONITOR_INGEST_HOSTNAME=server-monitor-ingest -SERVER_OPEN_TELEMETRY_INGEST_HOSTNAME=otel-telemetry-ingest +SERVER_TELEMETRY_HOSTNAME=telemetry SERVER_INCOMING_REQUEST_INGEST_HOSTNAME=incoming-request-ingest SERVER_TEST_SERVER_HOSTNAME=test-server SERVER_STATUS_PAGE_HOSTNAME=status-page @@ -116,7 +115,7 @@ SERVER_DOCS_HOSTNAME=docs APP_PORT=3002 PROBE_INGEST_PORT=3400 SERVER_MONITOR_INGEST_PORT=3404 -OPEN_TELEMETRY_INGEST_PORT=3403 +TELEMETRY_PORT=3403 INCOMING_REQUEST_INGEST_PORT=3402 TEST_SERVER_PORT=3800 ACCOUNTS_PORT=3003 @@ -244,7 +243,7 @@ WORKFLOW_TIMEOUT_IN_MS=5000 # Concurrency settings # Max number of telemetry jobs processed concurrently by OpenTelemetry Ingest worker -OPEN_TELEMETRY_INGEST_CONCURRENCY=100 +TELEMETRY_CONCURRENCY=100 # Max number of jobs processed concurrently by Fluent Logs worker FLUENT_LOGS_CONCURRENCY=100 @@ -313,7 +312,7 @@ LLM_SERVER_HUGGINGFACE_MODEL_NAME= DISABLE_TELEMETRY_FOR_ACCOUNTS=true DISABLE_TELEMETRY_FOR_APP=true DISABLE_TELEMETRY_FOR_PROBE_INGEST=true -DISABLE_TELEMETRY_FOR_OPEN_TELEMETRY_INGEST=true +DISABLE_TELEMETRY_FOR_TELEMETRY=true DISABLE_TELEMETRY_FOR_FLUENT_LOGS=true DISABLE_TELEMETRY_FOR_INCOMING_REQUEST_INGEST=true DISABLE_TELEMETRY_FOR_TEST_SERVER=true diff --git a/docker-compose.base.yml b/docker-compose.base.yml index c1ac07b72e..6eace9cea0 100644 --- a/docker-compose.base.yml +++ b/docker-compose.base.yml @@ -33,7 +33,7 @@ x-common-variables: &common-variables SERVER_APP_HOSTNAME: app SERVER_ALERT_HOSTNAME: alert SERVER_PROBE_INGEST_HOSTNAME: probe-ingest - SERVER_OPEN_TELEMETRY_INGEST_HOSTNAME: open-telemetry-ingest + SERVER_TELEMETRY_HOSTNAME: telemetry SERVER_INCOMING_REQUEST_INGEST_HOSTNAME: incoming-request-ingest SERVER_TEST_SERVER_HOSTNAME: test-server SERVER_STATUS_PAGE_HOSTNAME: status-page @@ -52,7 +52,7 @@ x-common-variables: &common-variables APP_PORT: ${APP_PORT} HOME_PORT: ${HOME_PORT} PROBE_INGEST_PORT: ${PROBE_INGEST_PORT} - OPEN_TELEMETRY_INGEST_PORT: ${OPEN_TELEMETRY_INGEST_PORT} + TELEMETRY_PORT: ${TELEMETRY_PORT} INCOMING_REQUEST_INGEST_PORT: ${INCOMING_REQUEST_INGEST_PORT} TEST_SERVER_PORT: ${TEST_SERVER_PORT} ACCOUNTS_PORT: ${ACCOUNTS_PORT} @@ -471,16 +471,16 @@ services: options: max-size: "1000m" - open-telemetry-ingest: + telemetry: networks: - oneuptime restart: always environment: <<: *common-runtime-variables - PORT: ${OPEN_TELEMETRY_INGEST_PORT} - DISABLE_TELEMETRY: ${DISABLE_TELEMETRY_FOR_OPEN_TELEMETRY_INGEST} + PORT: ${TELEMETRY_PORT} + DISABLE_TELEMETRY: ${DISABLE_TELEMETRY_FOR_TELEMETRY} # Max concurrent telemetry jobs the worker will process - OPEN_TELEMETRY_INGEST_CONCURRENCY: ${OPEN_TELEMETRY_INGEST_CONCURRENCY} + TELEMETRY_CONCURRENCY: ${TELEMETRY_CONCURRENCY} logging: driver: "local" options: diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 998f1212ff..e8cb118016 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -358,9 +358,9 @@ services: context: . dockerfile: ./ServerMonitorIngest/Dockerfile - open-telemetry-ingest: + telemetry: volumes: - - ./OpenTelemetryIngest:/usr/src/app:cached + - ./Telemetry:/usr/src/app:cached # Use node modules of the container and not host system. # https://stackoverflow.com/questions/29181032/add-a-volume-to-docker-but-exclude-a-sub-folder - /usr/src/app/node_modules/ @@ -370,11 +370,11 @@ services: - '9938:9229' # Debugging port. extends: file: ./docker-compose.base.yml - service: open-telemetry-ingest + service: telemetry build: network: host context: . - dockerfile: ./OpenTelemetryIngest/Dockerfile + dockerfile: ./Telemetry/Dockerfile incoming-request-ingest: volumes: diff --git a/docker-compose.yml b/docker-compose.yml index ca8300ef9b..94bad087b8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -114,11 +114,11 @@ services: file: ./docker-compose.base.yml service: server-monitor-ingest - open-telemetry-ingest: - image: oneuptime/open-telemetry-ingest:${APP_TAG} + telemetry: + image: oneuptime/telemetry:${APP_TAG} extends: file: ./docker-compose.base.yml - service: open-telemetry-ingest + service: telemetry incoming-request-ingest: image: oneuptime/incoming-request-ingest:${APP_TAG}