Migrate proto generation to buf

Bug: certusone/wormhole#282
Change-Id: Ib20d8b9bcf19a26b12a586532281d83e95f0653e

DEVELOP.md

@@ -74,3 +74,10 @@ To Solana:
 To Solana as CPI instruction:
 
     kubectl exec solana-devnet-0 -c setup -- client post-message --proxy CP1co2QMMoDPbsmV7PGcUTLFwyhgCgTXt25gLQ5LewE1 Bridge1p5gheXUvJ6jGWGeCsgPKgnE3YgdGKRVCMY9o 1 confirmed ffff
+
+
+## IntelliJ Protobuf Autocompletion
+
+Set the include path:
+
+![](https://i.imgur.com/bDij6Cu.png)

Tiltfile

@@ -30,15 +30,18 @@ def k8s_yaml_with_ns(objects):
 
 # protos
 
+proto_deps = ["./proto", "./generate-protos.sh", "buf.yaml", "buf.gen.yaml"]
+
 local_resource(
     name = "proto-gen",
-    deps = ["./proto", "./generate-protos.sh"],
+    deps = proto_deps,
     cmd = "./generate-protos.sh",
 )
 
 local_resource(
     name = "proto-gen-web",
-    deps = ["./proto", "./generate-protos-web.sh"],
+    deps = proto_deps,
+    resource_deps = ["proto-gen"],
     cmd = "./generate-protos-web.sh",
 )
 
@@ -67,6 +70,7 @@ k8s_yaml_with_ns(build_bridge_yaml())
 
 k8s_resource("guardian", resource_deps = ["proto-gen", "solana-devnet"], port_forwards = [
     port_forward(6060, name = "Debug/Status Server [:6060]"),
+    port_forward(7070, name = "Public RPC [:7070]"),
 ])
 
 # publicRPC proxy that allows grpc over http1, for local development
@@ -191,4 +195,4 @@ k8s_resource(
 k8s_resource(
     "terra-fcd",
     port_forwards = [port_forward(3060, name = "Terra FCD [:3060]")],
-)
+)

bridge/pkg/publicrpc/publicrpcserver.go

@@ -1,6 +1,7 @@
 package publicrpc
 
 import (
+	gossipv1 "github.com/certusone/wormhole/bridge/pkg/proto/gossip/v1"
 	publicrpcv1 "github.com/certusone/wormhole/bridge/pkg/proto/publicrpc/v1"
 	"go.uber.org/zap"
 )
@@ -23,7 +24,7 @@ func (s *PublicrpcServer) GetRawHeartbeats(req *publicrpcv1.GetRawHeartbeatsRequ
 	s.logger.Info("gRPC heartbeat stream opened by client")
 
 	// create a channel and register it for heartbeats
-	receiveChan := make(chan *publicrpcv1.Heartbeat, 50)
+	receiveChan := make(chan *gossipv1.Heartbeat, 50)
 	// clientId is the reference to the subscription that we will use for unsubscribing when the client disconnects.
 	clientId := s.rawHeartbeatListeners.subscribeHeartbeats(receiveChan)
 

bridge/pkg/publicrpc/rawheartbeats.go

@@ -1,14 +1,13 @@
 package publicrpc
 
 import (
+	gossipv1 "github.com/certusone/wormhole/bridge/pkg/proto/gossip/v1"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	"math/rand"
 	"sync"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"go.uber.org/zap"
-
-	publicrpcv1 "github.com/certusone/wormhole/bridge/pkg/proto/publicrpc/v1"
 )
 
 // track the number of active connections
@@ -24,13 +23,13 @@ var (
 // heartbeat messages to all the open connections.
 type RawHeartbeatConns struct {
 	mu     sync.RWMutex
-	subs   map[int]chan<- *publicrpcv1.Heartbeat
+	subs   map[int]chan<- *gossipv1.Heartbeat
 	logger *zap.Logger
 }
 
 func HeartbeatStreamMultiplexer(logger *zap.Logger) *RawHeartbeatConns {
 	ps := &RawHeartbeatConns{
-		subs:   map[int]chan<- *publicrpcv1.Heartbeat{},
+		subs:   map[int]chan<- *gossipv1.Heartbeat{},
 		logger: logger.Named("heartbeatmultiplexer"),
 	}
 	return ps
@@ -48,7 +47,7 @@ func (ps *RawHeartbeatConns) getUniqueClientId() int {
 }
 
 // subscribeHeartbeats adds a channel to the subscriber map, keyed by arbitrary clientId
-func (ps *RawHeartbeatConns) subscribeHeartbeats(ch chan *publicrpcv1.Heartbeat) int {
+func (ps *RawHeartbeatConns) subscribeHeartbeats(ch chan *gossipv1.Heartbeat) int {
 	ps.mu.Lock()
 	defer ps.mu.Unlock()
 
@@ -60,7 +59,7 @@ func (ps *RawHeartbeatConns) subscribeHeartbeats(ch chan *publicrpcv1.Heartbeat)
 }
 
 // PublishHeartbeat sends a message to all channels in the subscription map
-func (ps *RawHeartbeatConns) PublishHeartbeat(msg *publicrpcv1.Heartbeat) {
+func (ps *RawHeartbeatConns) PublishHeartbeat(msg *gossipv1.Heartbeat) {
 	ps.mu.RLock()
 	defer ps.mu.RUnlock()
 

buf.gen.web.yaml

@@ -0,0 +1,12 @@
+version: v1beta1
+plugins:
+  - name: tsproto
+    out: explorer/src/proto
+    path: tools/node_modules/.bin/protoc-gen-ts_proto
+    opt:
+      - paths=source_relative
+      - ts_proto_opt=esModuleInterop=true
+      - ts_proto_opt=env=browser
+      - ts_proto_opt=forceLong=string
+      - ts_proto_opt=outputClientImpl=grpc-web
+      - ts_proto_out=explorer/src/proto

buf.gen.yaml

@@ -0,0 +1,18 @@
+version: v1beta1
+plugins:
+  - name: go
+    out: bridge/pkg/proto
+    path: tools/bin/protoc-gen-go
+    opt:
+      - paths=source_relative
+  - name: go-grpc
+    out: bridge/pkg/proto
+    path: tools/bin/protoc-gen-go-grpc
+    opt:
+      - paths=source_relative
+  - name: grpc-gateway
+    out: bridge/pkg/proto
+    path: tools/bin/protoc-gen-grpc-gateway
+    opt:
+      - paths=source_relative
+      - generate_unbound_methods=true

buf.lock

@@ -0,0 +1,17 @@
+# Generated by buf. DO NOT EDIT.
+version: v1
+deps:
+  - remote: buf.build
+    owner: beta
+    repository: googleapis
+    branch: main
+    commit: 1c473ad9220a49bca9320f4cc690eba5
+    digest: b1-unlhrcI3tnJd0JEGuOb692LZ_tY_gCGq6mK1bgCn1Pg=
+    create_time: 2021-06-23T20:16:47.788079Z
+  - remote: buf.build
+    owner: grpc-ecosystem
+    repository: grpc-gateway
+    branch: main
+    commit: d19475fa22444a289c46af009acce62c
+    digest: b1-_zhDPyr_Ctc1QRAKuad6_0xvoyPd6QaB22ldm9gzS0Q=
+    create_time: 2021-04-26T15:19:26.742789Z

buf.yaml

@@ -0,0 +1,15 @@
+---
+version: v1beta1
+deps:
+  - buf.build/beta/googleapis
+  - buf.build/grpc-ecosystem/grpc-gateway
+build:
+  roots:
+    - proto
+lint:
+  use:
+    - BASIC
+    - FILE_LOWER_SNAKE_CASE
+breaking:
+  use:
+    - WIRE_JSON

buf.yml

@@ -1,10 +0,0 @@
-build:
-  roots:
-    - .
-lint:
-  use:
-    - BASIC
-    - FILE_LOWER_SNAKE_CASE
-breaking:
-  use:
-    - WIRE_JSON

generate-protos-web.sh

@@ -5,14 +5,7 @@
   npm ci
 )
 
+rm -rf explorer/src/proto
 mkdir -p explorer/src/proto
 
-tools/bin/buf protoc \
-  -Iproto \
-  -Ithird_party/googleapis \
-  --plugin tools/node_modules/.bin/protoc-gen-ts_proto \
-  --ts_proto_opt=esModuleInterop=true \
-  --ts_proto_opt=env=browser \
-  --ts_proto_opt=forceLong=string \
-  --ts_proto_opt=outputClientImpl=grpc-web \
-  --ts_proto_out=explorer/src/proto/ proto/**/**/**
+tools/bin/buf generate --template buf.gen.web.yaml

generate-protos.sh

@@ -1,27 +1,16 @@
 #!/usr/bin/env bash
+set -euo pipefail
 
 (
   cd tools/
   ./build.sh
 )
 
-(
-  cd third_party/
-  [[ ! -d googleapis ]] && git clone https://github.com/googleapis/googleapis
-  cd googleapis
-  git checkout 24fb9e5d1f37110bfa198189c34324aa3fdb0896
-)
+# TODO(leo): remove after a while
+rm -rf third_party/googleapis
 
-tools/bin/buf protoc \
-  -Iproto \
-  -Ithird_party/googleapis \
-  --plugin tools/bin/protoc-gen-go \
-  --go_opt=module=github.com/certusone/wormhole/bridge/pkg \
-  --go_out=bridge/pkg/ proto/**/**/**
+rm -rf bridge/pkg/proto
 
-tools/bin/buf protoc \
-  -Iproto \
-  -Ithird_party/googleapis \
-  --plugin tools/bin/protoc-gen-go-grpc \
-  --go-grpc_opt=module=github.com/certusone/wormhole/bridge/pkg \
-  --go-grpc_out=bridge/pkg/ proto/**/**/**
+tools/bin/buf mod update
+tools/bin/buf lint
+tools/bin/buf generate

proto/agent/v1/service.proto

@@ -1,51 +0,0 @@
-syntax = "proto3";
-
-package agent.v1;
-
-import "google/protobuf/timestamp.proto";
-// TODO: documentation
-
-option go_package = "github.com/certusone/wormhole/bridge/pkg/proto/agent/v1;agentv1";
-
-service Agent {
-  rpc SubmitVAA (SubmitVAARequest) returns (SubmitVAAResponse);
-  rpc GetBalance (GetBalanceRequest) returns (GetBalanceResponse);
-}
-
-message Empty {
-}
-
-message SubmitVAARequest {
-  VAA vaa = 1;
-  bool skip_preflight = 2;
-}
-
-message VAA {
-  uint32 Version = 1;
-  google.protobuf.Timestamp Timestamp = 2;
-  uint32 Nonce = 3;
-  uint32 EmitterChain = 4;
-  bytes EmitterAddress = 5;
-  uint64 Sequence = 6;
-  uint32 ConsistencyLevel = 7;
-  bytes Payload = 8;
-  uint32 GuardianSetIndex = 9;
-  repeated Signature Signatures = 10;
-}
-
-message Signature{
-  uint32 GuardianIndex = 1;
-  bytes Signature = 2;
-}
-
-message SubmitVAAResponse {
-  string signature = 1;
-}
-
-message GetBalanceRequest{
-
-}
-
-message GetBalanceResponse{
-  uint64 balance = 1;
-}

proto/gossip/v1/gossip.proto

@@ -2,7 +2,6 @@ syntax = "proto3";
 
 package gossip.v1;
 
-// full path of the resulting Go file is required in order to import in whisper.proto
 option go_package = "github.com/certusone/wormhole/bridge/pkg/proto/gossip/v1;gossipv1";
 
 message GossipMessage {

proto/node/v1/node.proto

@@ -4,8 +4,6 @@ package node.v1;
 
 option go_package = "github.com/certusone/wormhole/bridge/pkg/proto/node/v1;nodev1";
 
-import "google/api/annotations.proto";
-
 // NodePrivileged exposes an administrative API. It runs on a UNIX socket and is authenticated
 // using Linux filesystem permissions.
 service NodePrivileged {
@@ -67,7 +65,7 @@ message GuardianKey {
   // data is the binary representation of the secp256k1 private key.
   bytes data = 1;
   // Whether this key is deterministically generated and unsuitable for production mode.
-  bool unsafeDeterministicKey = 2;
+  bool unsafe_deterministic_key = 2;
 }
 
 // ContractUpgrade represents a Wormhole contract update to be submitted to and signed by the node.

proto/publicrpc/v1/publicrpc.proto

@@ -1,13 +1,10 @@
 syntax = "proto3";
 
-// only relevant for protobuf namespace
 package publicrpc.v1;
 
-// only relevant for Go namespace
 option go_package = "github.com/certusone/wormhole/bridge/pkg/proto/publicrpc/v1;publicrpcv1";
 
-// public import will include the required types in the Go output
-import public "gossip/v1/gossip.proto";
+import "gossip/v1/gossip.proto";
 
 // Publicrpc service exposes endpoints to be consumed externally; GUIs, historical record keeping, etc.
 service Publicrpc {