diff --git a/config/dashboard.toml b/config/dashboard.toml index 9bb2b6bf33..c00b167315 100644 --- a/config/dashboard.toml +++ b/config/dashboard.toml @@ -31,7 +31,7 @@ surcharge=false dispute_evidence_upload=false paypal_automatic_flow=false threeds_authenticator=false -global_search=false +global_search=true dispute_analytics=true configure_pmts=false branding=false diff --git a/config/prometheus.yaml b/config/prometheus.yaml index 574c9dcce7..85f6335c83 100644 --- a/config/prometheus.yaml +++ b/config/prometheus.yaml @@ -35,3 +35,11 @@ scrape_configs: static_configs: - targets: ["otel-collector:8888"] + + - job_name: "vector" + + # metrics_path defaults to '/metrics' + # scheme defaults to 'http'. + + static_configs: + - targets: ["vector:9598"] \ No newline at end of file diff --git a/config/vector.yaml b/config/vector.yaml new file mode 100644 index 0000000000..d8f5da3466 --- /dev/null +++ b/config/vector.yaml @@ -0,0 +1,134 @@ +acknowledgements: + enabled: true + +api: + enabled: true + address: 0.0.0.0:8686 + +sources: + kafka_tx_events: + type: kafka + bootstrap_servers: kafka0:29092 + group_id: sessionizer + topics: + - hyperswitch-payment-attempt-events + - hyperswitch-payment-intent-events + - hyperswitch-refund-events + - hyperswitch-dispute-events + decoding: + codec: json + + app_logs: + type: docker_logs + include_labels: + - "logs=promtail" + + vector_metrics: + type: internal_metrics + + node_metrics: + type: host_metrics + +transforms: + plus_1_events: + type: filter + inputs: + - kafka_tx_events + condition: ".sign_flag == 1" + + hs_server_logs: + type: filter + inputs: + - app_logs + condition: '.labels."com.docker.compose.service" == "hyperswitch-server"' + + parsed_hs_server_logs: + type: remap + inputs: + - app_logs + source: |- + .message = parse_json!(.message) + + events: + type: remap + inputs: + - plus_1_events + source: |- + .timestamp = from_unix_timestamp!(.created_at, unit: "seconds") + +sinks: + opensearch_events: + type: elasticsearch + inputs: + - events + endpoints: + - "https://opensearch:9200" + id_key: message_key + api_version: v7 + tls: + verify_certificate: false + verify_hostname: false + auth: + strategy: basic + user: admin + password: 0penS3arc# + encoding: + except_fields: + - message_key + - offset + - partition + - topic + bulk: + # Add a date prefixed index for better grouping + # index: "vector-{{ .topic }}-%Y-%m-%d" + index: "{{ .topic }}" + + opensearch_logs: + type: elasticsearch + inputs: + - parsed_hs_server_logs + endpoints: + - "https://opensearch:9200" + api_version: v7 + tls: + verify_certificate: false + verify_hostname: false + auth: + strategy: basic + user: admin + password: 0penS3arc# + bulk: + # Add a date prefixed index for better grouping + # index: "vector-{{ .topic }}-%Y-%m-%d" + index: "logs-{{ .container_name }}-%Y-%m-%d" + + log_events: + type: loki + inputs: + - kafka_tx_events + endpoint: http://loki:3100 + labels: + source: vector + topic: "{{ .topic }}" + job: kafka + encoding: + codec: json + + log_app_loki: + type: loki + inputs: + - parsed_hs_server_logs + endpoint: http://loki:3100 + labels: + source: vector + job: app_logs + container: "{{ .container_name }}" + stream: "{{ .stream }}" + encoding: + codec: json + + metrics: + type: prometheus_exporter + inputs: + - vector_metrics + - node_metrics diff --git a/crates/analytics/src/clickhouse.rs b/crates/analytics/src/clickhouse.rs index fd1746c4b9..47294c108f 100644 --- a/crates/analytics/src/clickhouse.rs +++ b/crates/analytics/src/clickhouse.rs @@ -262,6 +262,7 @@ impl TryInto for serde_json::Value { )) } } + impl TryInto for serde_json::Value { type Error = Report; diff --git a/docker-compose.yml b/docker-compose.yml index 57f7ccde1c..aff7ab54f4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -245,18 +245,6 @@ services: - ./config/grafana.ini:/etc/grafana/grafana.ini - ./config/grafana-datasource.yaml:/etc/grafana/provisioning/datasources/datasource.yml - promtail: - image: grafana/promtail:latest - volumes: - - ./logs:/var/log/router - - ./config:/etc/promtail - - /var/run/docker.sock:/var/run/docker.sock - command: -config.file=/etc/promtail/promtail.yaml - profiles: - - monitoring - networks: - - router_net - loki: image: grafana/loki:latest ports: @@ -387,21 +375,13 @@ services: soft: 262144 hard: 262144 - fluentd: - build: ./docker/fluentd - volumes: - - ./docker/fluentd/conf:/fluentd/etc - networks: - - router_net - profiles: - - olap - opensearch: - image: public.ecr.aws/opensearchproject/opensearch:1.3.14 + image: opensearchproject/opensearch:2 container_name: opensearch hostname: opensearch environment: - "discovery.type=single-node" + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=0penS3arc# profiles: - olap ports: @@ -410,7 +390,7 @@ services: - router_net opensearch-dashboards: - image: opensearchproject/opensearch-dashboards:1.3.14 + image: opensearchproject/opensearch-dashboards:2 ports: - 5601:5601 profiles: @@ -419,3 +399,18 @@ services: OPENSEARCH_HOSTS: '["https://opensearch:9200"]' networks: - router_net + + vector: + image: timberio/vector:latest-debian + ports: + - "8686" + - "9598" + profiles: + - olap + environment: + KAFKA_HOST: 'kafka0:29092' + networks: + - router_net + volumes: + - ./config/vector.yaml:/etc/vector/vector.yaml + - /var/run/docker.sock:/var/run/docker.sock \ No newline at end of file