rust/hyperswitch
1
0
Fork 0
mirror of https://github.com/juspay/hyperswitch.git synced 2025-10-27 19:46:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(analytics): added response to the connector outgoing event (#3129)

Browse Source 
Co-authored-by: harsh-sharma-juspay <125131007+harsh-sharma-juspay@users.noreply.github.com>
Co-authored-by: Sampras lopes <lsampras@pm.me>
This commit is contained in:
Sagar naik
2024-01-05 18:58:55 +05:30
committed by GitHub
parent 00008c16c1
commit d152c3a1ca
8 changed files with 145 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
crates/analytics/docs/clickhouse/scripts/api_events_v2.sql → crates/analytics/docs/clickhouse/scripts/api_events.sql
View File

@ -1,4 +1,4 @@
CREATE TABLE api_events_v2_queue (
CREATE TABLE api_events_queue (
`merchant_id` String,
`payment_id` Nullable(String),
`refund_id` Nullable(String),
@ -14,12 +14,15 @@ CREATE TABLE api_events_v2_queue (
`api_auth_type` LowCardinality(String),
`request` String,
`response` Nullable(String),
`error` Nullable(String),
`authentication_data` Nullable(String),
`status_code` UInt32,
`created_at` DateTime CODEC(T64, LZ4),
`created_at_timestamp` DateTime64(3),
`latency` UInt128,
`user_agent` String,
`ip_addr` String,
`hs_latency` Nullable(UInt128),
`http_method` LowCardinality(String),
`url_path` String
) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka0:29092',
kafka_topic_list = 'hyperswitch-api-log-events',
@ -28,7 +31,7 @@ kafka_format = 'JSONEachRow',
kafka_handle_error_mode = 'stream';
CREATE TABLE api_events_v2_dist (
CREATE TABLE api_events_dist (
`merchant_id` String,
`payment_id` Nullable(String),
`refund_id` Nullable(String),
@ -44,13 +47,15 @@ CREATE TABLE api_events_v2_dist (
`api_auth_type` LowCardinality(String),
`request` String,
`response` Nullable(String),
`error` Nullable(String),
`authentication_data` Nullable(String),
`status_code` UInt32,
`created_at` DateTime CODEC(T64, LZ4),
`inserted_at` DateTime CODEC(T64, LZ4),
`created_at_timestamp` DateTime64(3),
`latency` UInt128,
`user_agent` String,
`ip_addr` String,
`hs_latency` Nullable(UInt128),
`http_method` LowCardinality(String),
`url_path` String,
INDEX flowIndex flow_type TYPE bloom_filter GRANULARITY 1,
INDEX apiIndex api_flow TYPE bloom_filter GRANULARITY 1,
@ -62,7 +67,7 @@ ORDER BY
TTL created_at + toIntervalMonth(6)
;
CREATE MATERIALIZED VIEW api_events_v2_mv TO api_events_v2_dist (
CREATE MATERIALIZED VIEW api_events_mv TO api_events_dist (
`merchant_id` String,
`payment_id` Nullable(String),
`refund_id` Nullable(String),
@ -78,13 +83,15 @@ CREATE MATERIALIZED VIEW api_events_v2_mv TO api_events_v2_dist (
`api_auth_type` LowCardinality(String),
`request` String,
`response` Nullable(String),
`error` Nullable(String),
`authentication_data` Nullable(String),
`status_code` UInt32,
`created_at` DateTime CODEC(T64, LZ4),
`inserted_at` DateTime CODEC(T64, LZ4),
`created_at_timestamp` DateTime64(3),
`latency` UInt128,
`user_agent` String,
`ip_addr` String,
`hs_latency` Nullable(UInt128),
`http_method` LowCardinality(String),
`url_path` String
) AS
SELECT
@ -103,16 +110,19 @@ SELECT
api_auth_type,
request,
response,
error,
authentication_data,
status_code,
created_at,
created_at_timestamp,
now() as inserted_at,
latency,
user_agent,
ip_addr,
hs_latency,
http_method,
url_path
FROM
api_events_v2_queue
api_events_queue
where length(_error) = 0;
@ -133,6 +143,6 @@ SELECT
_offset AS offset,
_raw_message AS raw,
_error AS error
FROM api_events_v2_queue
FROM api_events_queue
WHERE length(_error) > 0
;

97
crates/analytics/docs/clickhouse/scripts/connector_events.sql Normal file
View File

@ -0,0 +1,97 @@
CREATE TABLE connector_events_queue (
`merchant_id` String,
`payment_id` Nullable(String),
`connector_name` LowCardinality(String),
`request_id` String,
`flow` LowCardinality(String),
`request` String,
`response` Nullable(String),
`error` Nullable(String),
`status_code` UInt32,
`created_at` DateTime64(3),
`latency` UInt128,
`method` LowCardinality(String)
) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka0:29092',
kafka_topic_list = 'hyperswitch-connector-api-events',
kafka_group_name = 'hyper-c1',
kafka_format = 'JSONEachRow',
kafka_handle_error_mode = 'stream';
CREATE TABLE connector_events_dist (
`merchant_id` String,
`payment_id` Nullable(String),
`connector_name` LowCardinality(String),
`request_id` String,
`flow` LowCardinality(String),
`request` String,
`response` Nullable(String),
`error` Nullable(String),
`status_code` UInt32,
`created_at` DateTime64(3),
`inserted_at` DateTime64(3),
`latency` UInt128,
`method` LowCardinality(String),
INDEX flowIndex flowTYPE bloom_filter GRANULARITY 1,
INDEX connectorIndex connector_name TYPE bloom_filter GRANULARITY 1,
INDEX statusIndex status_code TYPE bloom_filter GRANULARITY 1
) ENGINE = MergeTree
PARTITION BY toStartOfDay(created_at)
ORDER BY
(created_at, merchant_id, flow_type, status_code, api_flow)
TTL created_at + toIntervalMonth(6)
;
CREATE MATERIALIZED VIEW connector_events_mv TO connector_events_dist (
`merchant_id` String,
`payment_id` Nullable(String),
`connector_name` LowCardinality(String),
`request_id` String,
`flow` LowCardinality(String),
`request` String,
`response` Nullable(String),
`error` Nullable(String),
`status_code` UInt32,
`created_at` DateTime64(3),
`latency` UInt128,
`method` LowCardinality(String)
) AS
SELECT
merchant_id,
payment_id,
connector_name,
request_id,
flow,
request,
response,
error,
status_code,
created_at,
now() as inserted_at,
latency,
method,
FROM
connector_events_queue
where length(_error) = 0;
CREATE MATERIALIZED VIEW connector_events_parse_errors
(
`topic` String,
`partition` Int64,
`offset` Int64,
`raw` String,
`error` String
)
ENGINE = MergeTree
ORDER BY (topic, partition, offset)
SETTINGS index_granularity = 8192 AS
SELECT
_topic AS topic,
_partition AS partition,
_offset AS offset,
_raw_message AS raw,
_error AS error
FROM connector_events_queue
WHERE length(_error) > 0
;