feat(ci): add vector to handle logs pipeline (#5021)

config/dashboard.toml

@@ -31,7 +31,7 @@ surcharge=false
 dispute_evidence_upload=false
 paypal_automatic_flow=false
 threeds_authenticator=false
-global_search=false
+global_search=true
 dispute_analytics=true
 configure_pmts=false
 branding=false

config/prometheus.yaml

@@ -35,3 +35,11 @@ scrape_configs:
 
     static_configs:
       - targets: ["otel-collector:8888"]
+
+  - job_name: "vector"
+
+    # metrics_path defaults to '/metrics'
+    # scheme defaults to 'http'.
+
+    static_configs:
+      - targets: ["vector:9598"]

config/vector.yaml

@@ -0,0 +1,134 @@
+acknowledgements:
+  enabled: true
+
+api:
+  enabled: true
+  address: 0.0.0.0:8686
+
+sources:
+  kafka_tx_events:
+    type: kafka
+    bootstrap_servers: kafka0:29092
+    group_id: sessionizer
+    topics:
+      - hyperswitch-payment-attempt-events
+      - hyperswitch-payment-intent-events
+      - hyperswitch-refund-events
+      - hyperswitch-dispute-events
+    decoding:
+      codec: json
+
+  app_logs:
+    type: docker_logs
+    include_labels:
+      - "logs=promtail"
+
+  vector_metrics:
+    type: internal_metrics
+
+  node_metrics:
+    type: host_metrics
+
+transforms:
+  plus_1_events:
+    type: filter
+    inputs:
+      - kafka_tx_events
+    condition: ".sign_flag == 1"
+
+  hs_server_logs:
+    type: filter
+    inputs:
+      - app_logs
+    condition: '.labels."com.docker.compose.service" == "hyperswitch-server"'
+
+  parsed_hs_server_logs:
+    type: remap
+    inputs:
+      - app_logs
+    source: |-
+      .message = parse_json!(.message)
+
+  events:
+    type: remap
+    inputs:
+      - plus_1_events
+    source: |-
+      .timestamp = from_unix_timestamp!(.created_at, unit: "seconds")
+
+sinks:
+  opensearch_events:
+    type: elasticsearch
+    inputs:
+      - events
+    endpoints:
+      - "https://opensearch:9200"
+    id_key: message_key
+    api_version: v7
+    tls:
+      verify_certificate: false
+      verify_hostname: false
+    auth:
+      strategy: basic
+      user: admin
+      password: 0penS3arc#
+    encoding:
+      except_fields:
+        - message_key
+        - offset
+        - partition
+        - topic
+    bulk:
+      # Add a date prefixed index for better grouping
+      # index: "vector-{{ .topic }}-%Y-%m-%d"
+      index: "{{ .topic }}"
+
+  opensearch_logs:
+    type: elasticsearch
+    inputs:
+      - parsed_hs_server_logs
+    endpoints:
+      - "https://opensearch:9200"
+    api_version: v7
+    tls:
+      verify_certificate: false
+      verify_hostname: false
+    auth:
+      strategy: basic
+      user: admin
+      password: 0penS3arc#
+    bulk:
+      # Add a date prefixed index for better grouping
+      # index: "vector-{{ .topic }}-%Y-%m-%d"
+      index: "logs-{{ .container_name }}-%Y-%m-%d"
+
+  log_events:
+    type: loki
+    inputs:
+      - kafka_tx_events
+    endpoint: http://loki:3100
+    labels:
+      source: vector
+      topic: "{{ .topic }}"
+      job: kafka
+    encoding:
+      codec: json
+
+  log_app_loki:
+    type: loki
+    inputs:
+      - parsed_hs_server_logs
+    endpoint: http://loki:3100
+    labels:
+      source: vector
+      job: app_logs
+      container: "{{ .container_name }}"
+      stream: "{{ .stream }}"
+    encoding:
+      codec: json
+
+  metrics:
+    type: prometheus_exporter
+    inputs:
+      - vector_metrics
+      - node_metrics

crates/analytics/src/clickhouse.rs

@@ -262,6 +262,7 @@ impl TryInto<RefundFilterRow> for serde_json::Value {
         ))
     }
 }
+
 impl TryInto<DisputeMetricRow> for serde_json::Value {
     type Error = Report<ParsingError>;
 

docker-compose.yml

@@ -245,18 +245,6 @@ services:
       - ./config/grafana.ini:/etc/grafana/grafana.ini
       - ./config/grafana-datasource.yaml:/etc/grafana/provisioning/datasources/datasource.yml
 
-  promtail:
-    image: grafana/promtail:latest
-    volumes:
-      - ./logs:/var/log/router
-      - ./config:/etc/promtail
-      - /var/run/docker.sock:/var/run/docker.sock
-    command: -config.file=/etc/promtail/promtail.yaml
-    profiles:
-      - monitoring
-    networks:
-      - router_net
-
   loki:
     image: grafana/loki:latest
     ports:
@@ -387,21 +375,13 @@ services:
         soft: 262144
         hard: 262144
 
-  fluentd:
-    build: ./docker/fluentd
-    volumes:
-      - ./docker/fluentd/conf:/fluentd/etc
-    networks:
-      - router_net
-    profiles:
-      - olap
-
   opensearch:
-    image: public.ecr.aws/opensearchproject/opensearch:1.3.14
+    image: opensearchproject/opensearch:2
     container_name: opensearch
     hostname: opensearch
     environment:
       - "discovery.type=single-node"
+      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=0penS3arc#
     profiles:
       - olap
     ports:
@@ -410,7 +390,7 @@ services:
       - router_net
 
   opensearch-dashboards:
-    image: opensearchproject/opensearch-dashboards:1.3.14
+    image: opensearchproject/opensearch-dashboards:2
     ports:
       - 5601:5601
     profiles:
@@ -419,3 +399,18 @@ services:
       OPENSEARCH_HOSTS: '["https://opensearch:9200"]'
     networks:
       - router_net
+
+  vector:
+    image: timberio/vector:latest-debian
+    ports:
+      - "8686"
+      - "9598"
+    profiles:
+      - olap
+    environment:
+      KAFKA_HOST: 'kafka0:29092'
+    networks:
+      - router_net
+    volumes:
+      - ./config/vector.yaml:/etc/vector/vector.yaml
+      - /var/run/docker.sock:/var/run/docker.sock