// client starts an interop client to do stress test and a metrics server to report qps. package main import ( "flag" "fmt" "math/rand" "net" "strconv" "strings" "sync" "time" "golang.org/x/net/context" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/interop" testpb "google.golang.org/grpc/interop/grpc_testing" metricspb "google.golang.org/grpc/stress/grpc_testing" ) var ( serverAddresses = flag.String("server_addresses", "localhost:8080", "a list of server addresses") testCases = flag.String("test_cases", "", "a list of test cases along with the relative weights") testDurationSecs = flag.Int("test_duration_secs", -1, "test duration in seconds") numChannelsPerServer = flag.Int("num_channels_per_server", 1, "Number of channels (i.e connections) to each server") numStubsPerChannel = flag.Int("num_stubs_per_channel", 1, "Number of client stubs per each connection to server") metricsPort = flag.Int("metrics_port", 8081, "The port at which the stress client exposes QPS metrics") ) // testCaseWithWeight contains the test case type and its weight. type testCaseWithWeight struct { name string weight int } // parseTestCases converts test case string to a list of struct testCaseWithWeight. func parseTestCases(testCaseString string) ([]testCaseWithWeight, error) { testCaseStrings := strings.Split(testCaseString, ",") testCases := make([]testCaseWithWeight, len(testCaseStrings)) for i, str := range testCaseStrings { temp := strings.Split(str, ":") if len(temp) < 2 { return nil, fmt.Errorf("invalid test case with weight: %s", str) } // Check if test case is supported. switch temp[0] { case "empty_unary", "large_unary", "client_streaming", "server_streaming", "empty_stream": default: return nil, fmt.Errorf("unknown test type: %s", temp[0]) } testCases[i].name = temp[0] w, err := strconv.Atoi(temp[1]) if err != nil { return nil, fmt.Errorf("%v", err) } testCases[i].weight = w } return testCases, nil } // weightedRandomTestSelector defines a weighted random selector for test case types. type weightedRandomTestSelector struct { tests []testCaseWithWeight totalWeight int } // newWeightedRandomTestSelector constructs a weightedRandomTestSelector with the given list of testCaseWithWeight. func newWeightedRandomTestSelector(tests []testCaseWithWeight) *weightedRandomTestSelector { var totalWeight int for _, t := range tests { totalWeight += t.weight } rand.Seed(time.Now().UnixNano()) return &weightedRandomTestSelector{tests, totalWeight} } func (selector weightedRandomTestSelector) getNextTest() (string, error) { random := rand.Intn(selector.totalWeight) var weightSofar int for _, test := range selector.tests { weightSofar += test.weight if random < weightSofar { return test.name, nil } } return "", fmt.Errorf("no test case selected by weightedRandomTestSelector") } // gauge defines type for gauge. type gauge struct { mutex sync.RWMutex val int64 } // Set updates the gauge value func (g *gauge) set(v int64) { g.mutex.Lock() defer g.mutex.Unlock() g.val = v } func (g *gauge) get() int64 { g.mutex.RLock() defer g.mutex.RUnlock() return g.val } // Server implements metrics server functions. type server struct { mutex sync.RWMutex gauges map[string]*gauge } // newMetricsServer returns a new metrics server. func newMetricsServer() *server { return &server{gauges: make(map[string]*gauge)} } // GetAllGauges returns all gauges. func (s *server) GetAllGauges(in *metricspb.EmptyMessage, stream metricspb.MetricsService_GetAllGaugesServer) error { s.mutex.RLock() defer s.mutex.RUnlock() for name, gauge := range s.gauges { err := stream.Send(&metricspb.GaugeResponse{Name: name, Value: &metricspb.GaugeResponse_LongValue{gauge.get()}}) if err != nil { return err } } return nil } // GetGauge returns the gauge for the given name. func (s *server) GetGauge(ctx context.Context, in *metricspb.GaugeRequest) (*metricspb.GaugeResponse, error) { s.mutex.RLock() defer s.mutex.RUnlock() if g, ok := s.gauges[in.Name]; ok { return &metricspb.GaugeResponse{Name: in.Name, Value: &metricspb.GaugeResponse_LongValue{g.get()}}, nil } return nil, grpc.Errorf(codes.InvalidArgument, "gauge with name %s not found", in.Name) } // CreateGauge creates a guage using the given name in metrics server func (s *server) createGauge(name string) (*gauge, error) { s.mutex.Lock() defer s.mutex.Unlock() grpclog.Printf("create gauge: %s", name) if _, ok := s.gauges[name]; ok { // gauge already exists. return nil, fmt.Errorf("gauge %s already exists", name) } var g gauge s.gauges[name] = &g return &g, nil } func startServer(server *server, port int) { lis, err := net.Listen("tcp", ":"+strconv.Itoa(port)) if err != nil { grpclog.Fatalf("failed to listen: %v", err) } s := grpc.NewServer() metricspb.RegisterMetricsServiceServer(s, server) s.Serve(lis) } // stressClient defines client for stress test. type stressClient struct { testID int address string selector *weightedRandomTestSelector interopClient testpb.TestServiceClient stop <-chan bool } // newStressClient construct a new stressClient. func newStressClient(id int, addr string, conn *grpc.ClientConn, selector *weightedRandomTestSelector, stop <-chan bool) *stressClient { client := testpb.NewTestServiceClient(conn) return &stressClient{testID: id, address: addr, selector: selector, interopClient: client, stop: stop} } // mainLoop uses weightedRandomTestSelector to select test case and runs the tests. func (c *stressClient) mainLoop(gauge *gauge) { var numCalls int64 timeStarted := time.Now() for { done := make(chan bool) go func() { test, err := c.selector.getNextTest() if err != nil { grpclog.Printf("%v", err) done <- false } switch test { case "empty_unary": interop.DoEmptyUnaryCall(c.interopClient) case "large_unary": interop.DoLargeUnaryCall(c.interopClient) case "client_streaming": interop.DoClientStreaming(c.interopClient) case "server_streaming": interop.DoServerStreaming(c.interopClient) case "empty_stream": interop.DoEmptyStream(c.interopClient) default: grpclog.Fatalf("Unsupported test case: %d", test) } done <- true }() select { case <-c.stop: return case r := <-done: if r { numCalls++ gauge.set(int64(float64(numCalls) / time.Since(timeStarted).Seconds())) } } } } func logParameterInfo(addresses []string, tests []testCaseWithWeight) { grpclog.Printf("server_addresses: %s", *serverAddresses) grpclog.Printf("test_cases: %s", *testCases) grpclog.Printf("test_duration-secs: %d", *testDurationSecs) grpclog.Printf("num_channels_per_server: %d", *numChannelsPerServer) grpclog.Printf("num_stubs_per_channel: %d", *numStubsPerChannel) grpclog.Printf("metrics_port: %d", *metricsPort) grpclog.Println("addresses:") for i, addr := range addresses { grpclog.Printf("%d. %s\n", i+1, addr) } grpclog.Println("tests:") for i, test := range tests { grpclog.Printf("%d. %v\n", i+1, test) } } func main() { flag.Parse() addresses := strings.Split(*serverAddresses, ",") tests, err := parseTestCases(*testCases) if err != nil { grpclog.Fatalf("%v\n", err) } logParameterInfo(addresses, tests) testSelector := newWeightedRandomTestSelector(tests) metricsServer := newMetricsServer() var wg sync.WaitGroup wg.Add(len(addresses) * *numChannelsPerServer * *numStubsPerChannel) stop := make(chan bool) var clientIndex int for serverIndex, address := range addresses { for connIndex := 0; connIndex < *numChannelsPerServer; connIndex++ { conn, err := grpc.Dial(address, grpc.WithInsecure()) if err != nil { grpclog.Fatalf("Fail to dial: %v", err) } defer conn.Close() for stubIndex := 0; stubIndex < *numStubsPerChannel; stubIndex++ { clientIndex++ client := newStressClient(clientIndex, address, conn, testSelector, stop) buf := fmt.Sprintf("/stress_test/server_%d/channel_%d/stub_%d/qps", serverIndex+1, connIndex+1, stubIndex+1) go func() { defer wg.Done() if g, err := metricsServer.createGauge(buf); err != nil { grpclog.Fatalf("%v", err) } else { client.mainLoop(g) } }() } } } go startServer(metricsServer, *metricsPort) if *testDurationSecs > 0 { time.Sleep(time.Duration(*testDurationSecs) * time.Second) close(stop) } wg.Wait() grpclog.Printf(" ===== ALL DONE ===== ") }