Node/CCQ: Add rate limiting to proxy (#4080)

* Node/CCQ: Add rate limiting

* Code review rework

* Node/CCQ: Make burst size default to one not zero

* Tweak description of burst size in doc

devnet/query-server.yaml

@@ -60,7 +60,6 @@ spec:
             - --logLevel=warn
             - --shutdownDelay1
             - "0"
-            - --allowAnything
           ports:
             - containerPort: 6069
               name: rest

docs/query_proxy.md

@@ -62,7 +62,6 @@ Optional Parameters
 
 - The `gossipAdvertiseAddress` argument allows you to specify an external IP to advertize on P2P (use if behind a NAT or running in k8s).
 - The `monitorPeers` flag will cause the proxy server to periodically check its connectivity to the P2P bootstrap peers, and attempt to reconnect if necessary.
-- The `allowAnything` flag enables defining users with the `allowAnything` flag set to true. This is only allowed in testnet and devnet.
 
 #### Creating the Signing Key File
 
@@ -96,6 +95,9 @@ The simplest file would look something like this
 
 ```json
 {
+  "allowAnythingSupported": false,
+  "defaultRateLimit": 0.5,
+  "defaultBurstSize": 1,
   "permissions": [
     {
       "userName": "Monitor",
@@ -162,8 +164,10 @@ as soon as you save the file, the changes will be picked up (whether they are lo
 
 #### The `allowAnything` flag
 
+The `allowAnything` flag may only be specified for a user if you are running in testnet and the `allowAnythingSupported` flag in the
+permissions file is set to true.
+
 If this flag is specified for a user, then that user may make any call on any supported chain, without restriction.
-This flag is only allowed if the `allowAnything` command line argument is specified.
 If this flag is specified, then `allowedCalls` must not be specified.
 
 ```json
@@ -179,6 +183,22 @@ If this flag is specified, then `allowedCalls` must not be specified.
 }
 ```
 
+### Rate Limiting
+
+The query proxy server supports rate limiting by specifying two parameters. The rate limit, which is a floating point value, and the burst size,
+which is an int. See [here](https://pkg.go.dev/golang.org/x/time/rate#Limiter) for a description of how the rate limiter works.
+
+Note that if the rate limits are not specified, or the rate is set to zero, rate limiting will be disabled, allowing unlimited queries per second. The burst size only has meaning if the rate limit is specified. It defaults to one, and zero is not a valid value.
+
+The rate limits may be specified at either of two levels.
+
+First, you may specify global defaults for rate limiting by specifying the `defaultRateLimit` and `defaultBurstSize` parameters
+in the permissions file. If these parameters are specified, they apply to all users for which per-user parameters are not specified.
+This means that each of these users will be allowed that many queries per second.
+
+Second, you may override the global defaults for a given user by specifying `rateLimit` and `burstSize` for that user. Also note that
+you can disable rate limits for a given user (overriding the default) by setting their `rateLimit` to zero.
+
 ### Validating Permissions File Changes
 
 The query server automatically detects changes to the permissions file and attempts to reload them. If there are errors in the updated
@@ -188,12 +208,10 @@ the server from coming up on the next restart. You can avoid this problem by ver
 To do this, you can copy the permissions file to some other file, make your changes to the copy, and then do the following:
 
 ```sh
-$ guardiand query-server --verifyPermissions --permFile new.permissions.file.json --allowAnything
+$ guardiand query-server --env mainnet --verifyPermissions --permFile new.permissions.file.json
 ```
 
-where `new.permissions.file.json` is the path to the updated file. Additionally, if your permission file includes the `allowAnything`
-flag for any of the users, you must specify that flag on the command line when doing the verify.
-
+where the `--env` flag should be either `mainnet` or `testnet` and `new.permissions.file.json` is the path to the updated file.
 If the updated file is good, the program will exit immediately with no output and an exit code of zero. If the file contains
 errors, the first error will be printed, and the exit code will be one.
 

node/cmd/ccq/devnet.permissions.json

@@ -1,4 +1,5 @@
 {
+  "allowAnythingSupported": true,
   "permissions": [
     {
       "userName": "Test User",
@@ -184,6 +185,14 @@
       "apiKey": "my_secret_key_3",
       "allowUnsigned": true,
       "allowAnything": true
+    },
+    {
+      "userName": "Rate Limited User",
+      "apiKey": "rate_limited_key",
+      "rateLimit": 1.0,
+      "burstSize": 2,
+      "allowUnsigned": true,
+      "allowAnything": true
     }
   ]
 }

node/cmd/ccq/http.go

@@ -87,6 +87,14 @@ func (s *httpServer) handleQuery(w http.ResponseWriter, r *http.Request) {
 		invalidQueryRequestReceived.WithLabelValues("invalid_api_key").Inc()
 		return
 	}
+
+	if permEntry.rateLimiter != nil && !permEntry.rateLimiter.Allow() {
+		s.logger.Debug("denying request due to rate limit", zap.String("userId", permEntry.userName))
+		http.Error(w, "rate limit exceeded", http.StatusTooManyRequests)
+		rateLimitExceededByUser.WithLabelValues(permEntry.userName).Inc()
+		return
+	}
+
 	totalRequestsByUser.WithLabelValues(permEntry.userName).Inc()
 
 	queryRequestBytes, err := hex.DecodeString(q.Bytes)

node/cmd/ccq/metrics.go

@@ -45,6 +45,12 @@ var (
 			Help: "Total number of successful queries by user name",
 		}, []string{"user_name"})
 
+	rateLimitExceededByUser = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "ccq_server_rate_limit_exceeded_by_user",
+			Help: "Total number of queries rejected due to rate limiting per user name",
+		}, []string{"user_name"})
+
 	failedQueriesByUser = promauto.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "ccq_server_failed_queries_by_user",

node/cmd/ccq/parse_config_test.go

@@ -5,15 +5,17 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/certusone/wormhole/node/pkg/common"
 	"github.com/certusone/wormhole/node/pkg/query"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/wormhole-foundation/wormhole/sdk/vaa"
 	"go.uber.org/zap"
+	"golang.org/x/time/rate"
 )
 
 func TestParseConfigFileDoesntExist(t *testing.T) {
-	_, err := parseConfigFile("missingFile.json", false)
+	_, err := parseConfigFile("missingFile.json", common.MainNet)
 	require.Error(t, err)
 	assert.Equal(t, `failed to open permissions file "missingFile.json": open missingFile.json: no such file or directory`, err.Error())
 }
@@ -52,7 +54,7 @@ func TestParseConfigBadJson(t *testing.T) {
       ]
     }`
 
-	_, err := parseConfig([]byte(str), false)
+	_, err := parseConfig([]byte(str), common.MainNet)
 	require.Error(t, err)
 	assert.Equal(t, `failed to unmarshal json: unexpected end of JSON input`, err.Error())
 }
@@ -93,7 +95,7 @@ func TestParseConfigDuplicateUser(t *testing.T) {
   ]
 }`
 
-	_, err := parseConfig([]byte(str), false)
+	_, err := parseConfig([]byte(str), common.MainNet)
 	require.Error(t, err)
 	assert.Equal(t, `UserName "Test User" is a duplicate`, err.Error())
 }
@@ -134,7 +136,7 @@ func TestParseConfigDuplicateApiKey(t *testing.T) {
   ]
 }`
 
-	_, err := parseConfig([]byte(str), false)
+	_, err := parseConfig([]byte(str), common.MainNet)
 	require.Error(t, err)
 	assert.Equal(t, `API key "my_secret_key" is a duplicate`, err.Error())
 }
@@ -160,7 +162,7 @@ func TestParseConfigUnsupportedCallType(t *testing.T) {
   ]
 }`
 
-	_, err := parseConfig([]byte(str), false)
+	_, err := parseConfig([]byte(str), common.MainNet)
 	require.Error(t, err)
 	assert.Equal(t, `unsupported call type for user "Test User", must be "ethCall", "ethCallByTimestamp", "ethCallWithFinality", "solAccount" or "solPDA"`, err.Error())
 }
@@ -186,7 +188,7 @@ func TestParseConfigInvalidContractAddress(t *testing.T) {
   ]
 }`
 
-	_, err := parseConfig([]byte(str), false)
+	_, err := parseConfig([]byte(str), common.MainNet)
 	require.Error(t, err)
 	assert.Equal(t, `invalid contract address "HelloWorld" for user "Test User"`, err.Error())
 }
@@ -212,7 +214,7 @@ func TestParseConfigInvalidEthCall(t *testing.T) {
   ]
 }`
 
-	_, err := parseConfig([]byte(str), false)
+	_, err := parseConfig([]byte(str), common.MainNet)
 	require.Error(t, err)
 	assert.Equal(t, `invalid eth call "HelloWorld" for user "Test User"`, err.Error())
 }
@@ -238,7 +240,7 @@ func TestParseConfigInvalidEthCallLength(t *testing.T) {
   ]
 }`
 
-	_, err := parseConfig([]byte(str), false)
+	_, err := parseConfig([]byte(str), common.MainNet)
 	require.Error(t, err)
 	assert.Equal(t, `eth call "0x06fd" for user "Test User" has an invalid length, must be 4 bytes`, err.Error())
 }
@@ -272,7 +274,7 @@ func TestParseConfigDuplicateAllowedCallForUser(t *testing.T) {
   ]
 }`
 
-	_, err := parseConfig([]byte(str), false)
+	_, err := parseConfig([]byte(str), common.MainNet)
 	require.Error(t, err)
 	assert.Equal(t, `"ethCall:2:000000000000000000000000b4fbf271143f4fbf7b91a5ded31805e42b2208d6:06fdde03" is a duplicate allowed call for user "Test User"`, err.Error())
 }
@@ -328,7 +330,7 @@ func TestParseConfigSuccess(t *testing.T) {
   ]
 }`
 
-	perms, err := parseConfig([]byte(str), false)
+	perms, err := parseConfig([]byte(str), common.MainNet)
 	require.NoError(t, err)
 	assert.Equal(t, 1, len(perms))
 
@@ -353,9 +355,42 @@ func TestParseConfigSuccess(t *testing.T) {
 	assert.True(t, exists)
 }
 
+func TestParseConfigAllowAnythingWhenNotSpecified(t *testing.T) {
+	str := `
+	{
+  "permissions": [
+    {
+      "userName": "Test User",
+      "apiKey": "my_secret_key",
+      "allowedCalls": [
+        {
+          "ethCall": {
+            "note:": "Name of WETH on Goerli",
+            "chain": 2,
+            "contractAddress": "B4FBF271143F4FBf7B91A5ded31805e42b2208d6",
+            "call": "0x06fdde03"
+          }
+        }
+      ]
+    },
+    {
+      "userName": "Test User2",
+      "apiKey": "my_secret_key_2",
+      "allowUnsigned": true,
+      "allowAnything": true
+    }
+  ]
+}`
+
+	_, err := parseConfig([]byte(str), common.TestNet)
+	require.Error(t, err)
+	assert.Equal(t, `UserName "Test User2" has "allowAnything" specified when the feature is not enabled`, err.Error())
+}
+
 func TestParseConfigAllowAnythingWhenNotEnabled(t *testing.T) {
 	str := `
 	{
+  "AllowAnythingSupported": false,
   "permissions": [
     {
       "userName": "Test User",
@@ -380,7 +415,7 @@ func TestParseConfigAllowAnythingWhenNotEnabled(t *testing.T) {
   ]
 }`
 
-	_, err := parseConfig([]byte(str), false)
+	_, err := parseConfig([]byte(str), common.TestNet)
 	require.Error(t, err)
 	assert.Equal(t, `UserName "Test User2" has "allowAnything" specified when the feature is not enabled`, err.Error())
 }
@@ -388,6 +423,7 @@ func TestParseConfigAllowAnythingWhenNotEnabled(t *testing.T) {
 func TestParseConfigAllowAnythingWithAllowedCallsIsInvalid(t *testing.T) {
 	str := `
 	{
+  "allowAnythingSupported": true,
   "permissions": [
     {
       "userName": "Test User",
@@ -422,14 +458,47 @@ func TestParseConfigAllowAnythingWithAllowedCallsIsInvalid(t *testing.T) {
   ]
 }`
 
-	_, err := parseConfig([]byte(str), true)
+	_, err := parseConfig([]byte(str), common.TestNet)
 	require.Error(t, err)
 	assert.Equal(t, `UserName "Test User2" has "allowedCalls" specified with "allowAnything", which is not allowed`, err.Error())
 }
 
+func TestParseConfigAllowAnythingNotAllowedInMainnet(t *testing.T) {
+	str := `
+	{
+  "allowAnythingSupported": true,
+  "permissions": [
+    {
+      "userName": "Test User",
+      "apiKey": "my_secret_key",
+      "allowedCalls": [
+        {
+          "ethCall": {
+            "note:": "Name of WETH on Goerli",
+            "chain": 2,
+            "contractAddress": "B4FBF271143F4FBf7B91A5ded31805e42b2208d6",
+            "call": "0x06fdde03"
+          }
+        }
+      ]
+    },
+    {
+      "userName": "Test User2",
+      "apiKey": "my_secret_key_2",
+      "allowUnsigned": true,
+      "allowAnything": true
+    }
+  ]
+}`
+
+	_, err := parseConfig([]byte(str), common.MainNet)
+	require.Equal(t, `the "allowAnythingSupported" flag is not supported in mainnet`, err.Error())
+}
+
 func TestParseConfigAllowAnythingSuccess(t *testing.T) {
 	str := `
 	{
+  "allowAnythingSupported": true,
   "permissions": [
     {
       "userName": "Test User",
@@ -454,7 +523,7 @@ func TestParseConfigAllowAnythingSuccess(t *testing.T) {
   ]
 }`
 
-	perms, err := parseConfig([]byte(str), true)
+	perms, err := parseConfig([]byte(str), common.TestNet)
 	require.NoError(t, err)
 	assert.Equal(t, 2, len(perms))
 
@@ -496,7 +565,7 @@ func TestParseConfigContractWildcard(t *testing.T) {
   ]
 }`
 
-	perms, err := parseConfig([]byte(str), true)
+	perms, err := parseConfig([]byte(str), common.MainNet)
 	require.NoError(t, err)
 	assert.Equal(t, 1, len(perms))
 
@@ -618,3 +687,276 @@ func createCallData(t *testing.T, toStr string, dataStr string) []*query.EthCall
 		},
 	}
 }
+
+func TestParseConfigWithRateLimiterNoDefaults(t *testing.T) {
+	str := `
+	{
+  "permissions": [
+    {
+      "userName": "Test user without rate limits",
+      "apiKey": "my_secret_key_without_rate_limits",
+      "allowedCalls": [
+        {
+          "ethCall": {
+            "note:": "Name of WETH on Goerli",
+            "chain": 2,
+            "contractAddress": "B4FBF271143F4FBf7B91A5ded31805e42b2208d6",
+            "call": "0x06fdde03"
+          }
+        }
+      ]
+    },
+    {
+      "userName": "Test user with rate limits",
+      "apiKey": "my_secret_key_with_rate_limits",
+      "rateLimit": 0.5,
+      "burstSize": 1,
+      "allowedCalls": [
+        {
+          "ethCall": {
+            "note:": "Name of WETH on Goerli",
+            "chain": 2,
+            "contractAddress": "B4FBF271143F4FBf7B91A5ded31805e42b2208d6",
+            "call": "0x06fdde03"
+          }
+        }
+      ]
+    }
+  ]
+}`
+
+	perms, err := parseConfig([]byte(str), common.MainNet)
+	require.NoError(t, err)
+	assert.Equal(t, 2, len(perms))
+
+	perm, exists := perms["my_secret_key_without_rate_limits"]
+	require.True(t, exists)
+	assert.Nil(t, perm.rateLimiter)
+
+	perm, exists = perms["my_secret_key_with_rate_limits"]
+	require.True(t, exists)
+	require.NotNil(t, perm.rateLimiter)
+	assert.Equal(t, rate.Limit(0.5), perm.rateLimiter.Limit())
+	assert.Equal(t, 1, perm.rateLimiter.Burst())
+}
+
+func TestParseConfigWithRateLimiterWithDefaults(t *testing.T) {
+	str := `
+	{
+  "defaultRateLimit": 0.5,
+  "defaultBurstSize": 1,  
+  "permissions": [
+    {
+      "userName": "Test user using default rate limits",
+      "apiKey": "my_secret_key_using_default_rate_limits",
+      "allowedCalls": [
+        {
+          "ethCall": {
+            "note:": "Name of WETH on Goerli",
+            "chain": 2,
+            "contractAddress": "B4FBF271143F4FBf7B91A5ded31805e42b2208d6",
+            "call": "0x06fdde03"
+          }
+        }
+      ]
+    },
+    {
+      "userName": "Test user overriding default rate limits",
+      "apiKey": "my_secret_key_overriding_default_rate_limits",
+      "rateLimit": 1,
+      "burstSize": 2,
+      "allowedCalls": [
+        {
+          "ethCall": {
+            "note:": "Name of WETH on Goerli",
+            "chain": 2,
+            "contractAddress": "B4FBF271143F4FBf7B91A5ded31805e42b2208d6",
+            "call": "0x06fdde03"
+          }
+        }
+      ]
+    },
+    {
+      "userName": "Test user disabling rate limits",
+      "apiKey": "my_secret_key_disabling_rate_limits",
+      "rateLimit": 0,
+      "allowedCalls": [
+        {
+          "ethCall": {
+            "note:": "Name of WETH on Goerli",
+            "chain": 2,
+            "contractAddress": "B4FBF271143F4FBf7B91A5ded31805e42b2208d6",
+            "call": "0x06fdde03"
+          }
+        }
+      ]
+    }
+  ]
+}`
+
+	perms, err := parseConfig([]byte(str), common.MainNet)
+	require.NoError(t, err)
+	assert.Equal(t, 3, len(perms))
+
+	perm, exists := perms["my_secret_key_using_default_rate_limits"]
+	require.True(t, exists)
+	require.NotNil(t, perm.rateLimiter)
+	assert.Equal(t, rate.Limit(0.5), perm.rateLimiter.Limit())
+	assert.Equal(t, 1, perm.rateLimiter.Burst())
+
+	perm, exists = perms["my_secret_key_overriding_default_rate_limits"]
+	require.True(t, exists)
+	require.NotNil(t, perm.rateLimiter)
+	assert.Equal(t, rate.Limit(1.0), perm.rateLimiter.Limit())
+	assert.Equal(t, 2, perm.rateLimiter.Burst())
+
+	perm, exists = perms["my_secret_key_disabling_rate_limits"]
+	require.True(t, exists)
+	require.Nil(t, perm.rateLimiter)
+}
+
+func TestParseConfigWithRateLimiterPerUser(t *testing.T) {
+	str := `
+	{
+  "defaultRateLimit": 0.5,
+  "defaultBurstSize": 1,
+  "permissions": [
+    {
+      "userName": "Test User",
+      "apiKey": "My_secret_key",
+      "rateLimit": 1.5,
+      "burstSize": 3,      
+      "allowedCalls": [
+        {
+          "ethCall": {
+            "note:": "Name of WETH on Goerli",
+            "chain": 2,
+            "contractAddress": "B4FBF271143F4FBf7B91A5ded31805e42b2208d6",
+            "call": "0x06fdde03"
+          }
+        }
+      ]
+    },
+    {
+      "userName": "Test User 2",
+      "apiKey": "My_secret_key_2",
+      "allowedCalls": [
+        {
+          "ethCall": {
+            "note:": "Name of WETH on Goerli",
+            "chain": 2,
+            "contractAddress": "B4FBF271143F4FBf7B91A5ded31805e42b2208d6",
+            "call": "0x06fdde03"
+          }
+        }
+      ]
+    }
+  ]
+}`
+
+	perms, err := parseConfig([]byte(str), common.MainNet)
+	require.NoError(t, err)
+	assert.Equal(t, 2, len(perms))
+
+	perm, exists := perms["my_secret_key"]
+	require.True(t, exists)
+
+	require.NotNil(t, perm.rateLimiter)
+	assert.Equal(t, rate.Limit(1.5), perm.rateLimiter.Limit())
+	assert.Equal(t, 3, perm.rateLimiter.Burst())
+
+	perm, exists = perms["my_secret_key_2"]
+	require.True(t, exists)
+
+	require.NotNil(t, perm.rateLimiter)
+	assert.Equal(t, rate.Limit(0.5), perm.rateLimiter.Limit())
+	assert.Equal(t, 1, perm.rateLimiter.Burst())
+}
+
+func TestParseConfigWithRateLimiterButDefaultBurstSizeNotSet(t *testing.T) {
+	str := `
+	{
+  "defaultRateLimit": 0.5,
+  "permissions": [
+    {
+      "userName": "Test user using default rate limits",
+      "apiKey": "my_secret_key_using_default_rate_limits",
+      "allowedCalls": [
+        {
+          "ethCall": {
+            "note:": "Name of WETH on Goerli",
+            "chain": 2,
+            "contractAddress": "B4FBF271143F4FBf7B91A5ded31805e42b2208d6",
+            "call": "0x06fdde03"
+          }
+        }
+      ]
+    }
+  ]
+}`
+
+	perms, err := parseConfig([]byte(str), common.MainNet)
+	require.NoError(t, err)
+	assert.Equal(t, 1, len(perms))
+
+	perm, exists := perms["my_secret_key_using_default_rate_limits"]
+	require.True(t, exists)
+	require.NotNil(t, perm.rateLimiter)
+	assert.Equal(t, rate.Limit(0.5), perm.rateLimiter.Limit())
+	assert.Equal(t, 1, perm.rateLimiter.Burst())
+}
+
+func TestParseConfigWithRateLimiterButDefaultBurstSizeNIsSetToZero(t *testing.T) {
+	str := `
+	{
+  "defaultBurstSize": 0,
+  "permissions": [
+    {
+      "userName": "Test user using default rate limits",
+      "apiKey": "my_secret_key_using_default_rate_limits",
+      "allowedCalls": [
+        {
+          "ethCall": {
+            "note:": "Name of WETH on Goerli",
+            "chain": 2,
+            "contractAddress": "B4FBF271143F4FBf7B91A5ded31805e42b2208d6",
+            "call": "0x06fdde03"
+          }
+        }
+      ]
+    }
+  ]
+}`
+
+	_, err := parseConfig([]byte(str), common.MainNet)
+	assert.Equal(t, "the default burst size may not be zero", err.Error())
+}
+
+func TestParseConfigWithRateLimiterButPerUserBurstSizeSetToZero(t *testing.T) {
+	str := `
+	{
+  "defaultRateLimit": 0.5,
+  "defaultBurstSize": 1,
+  "permissions": [
+    {
+      "userName": "Test user overriding default rate limits",
+      "apiKey": "my_secret_key_overriding_default_rate_limits",
+      "rateLimit": 1,
+      "burstSize": 0,
+      "allowedCalls": [
+        {
+          "ethCall": {
+            "note:": "Name of WETH on Goerli",
+            "chain": 2,
+            "contractAddress": "B4FBF271143F4FBf7B91A5ded31805e42b2208d6",
+            "call": "0x06fdde03"
+          }
+        }
+      ]
+    }
+  ]
+}`
+
+	_, err := parseConfig([]byte(str), common.MainNet)
+	assert.Equal(t, "if rate limiting is enabled, the burst size may not be zero", err.Error())
+}

node/cmd/ccq/permissions.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -14,6 +15,7 @@ import (
 	"github.com/certusone/wormhole/node/pkg/query"
 	"github.com/wormhole-foundation/wormhole/sdk/vaa"
 	"go.uber.org/zap"
+	"golang.org/x/time/rate"
 
 	"github.com/gagliardetto/solana-go"
 	"gopkg.in/godo.v2/watcher/fswatch"
@@ -21,7 +23,10 @@ import (
 
 type (
 	Config struct {
-		Permissions []User `json:"Permissions"`
+		AllowAnythingSupported bool    `json:"AllowAnythingSupported"`
+		DefaultRateLimit       float64 `json:"DefaultRateLimit"`
+		DefaultBurstSize       int     `json:"DefaultBurstSize"`
+		Permissions            []User  `json:"Permissions"`
 	}
 
 	User struct {
@@ -29,6 +34,8 @@ type (
 		ApiKey        string        `json:"apiKey"`
 		AllowUnsigned bool          `json:"allowUnsigned"`
 		AllowAnything bool          `json:"allowAnything"`
+		RateLimit     *float64      `json:"RateLimit"`
+		BurstSize     *int          `json:"BurstSize"`
 		LogResponses  bool          `json:"logResponses"`
 		AllowedCalls  []AllowedCall `json:"allowedCalls"`
 	}
@@ -75,6 +82,7 @@ type (
 	permissionEntry struct {
 		userName      string
 		apiKey        string
+		rateLimiter   *rate.Limiter
 		allowUnsigned bool
 		allowAnything bool
 		logResponses  bool
@@ -84,25 +92,24 @@ type (
 	allowedCallsForUser map[string]struct{}
 
 	Permissions struct {
-		lock          sync.Mutex
-		permMap       PermissionsMap
-		fileName      string
-		allowAnything bool
-		watcher       *fswatch.Watcher
+		lock     sync.Mutex
+		env      common.Environment
+		permMap  PermissionsMap
+		fileName string
+		watcher  *fswatch.Watcher
 	}
 )
 
 // NewPermissions creates a Permissions object which contains the per-user permissions.
-func NewPermissions(fileName string, allowAnything bool) (*Permissions, error) {
-	permMap, err := parseConfigFile(fileName, allowAnything)
+func NewPermissions(fileName string, env common.Environment) (*Permissions, error) {
+	permMap, err := parseConfigFile(fileName, env)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Permissions{
-		permMap:       permMap,
-		fileName:      fileName,
-		allowAnything: allowAnything,
+		permMap:  permMap,
+		fileName: fileName,
 	}, nil
 }
 
@@ -131,7 +138,7 @@ func (perms *Permissions) StartWatcher(ctx context.Context, logger *zap.Logger,
 
 // Reload reloads the permissions file.
 func (perms *Permissions) Reload(logger *zap.Logger) {
-	permMap, err := parseConfigFile(perms.fileName, perms.allowAnything)
+	permMap, err := parseConfigFile(perms.fileName, perms.env)
 	if err != nil {
 		logger.Error("failed to reload the permissions file, sticking with the old one", zap.String("fileName", perms.fileName), zap.Error(err))
 		permissionFileReloadsFailure.Inc()
@@ -163,7 +170,7 @@ func (perms *Permissions) GetUserEntry(apiKey string) (*permissionEntry, bool) {
 const ETH_CALL_SIG_LENGTH = 4
 
 // parseConfigFile parses the permissions config file into a map keyed by API key.
-func parseConfigFile(fileName string, allowAnything bool) (PermissionsMap, error) {
+func parseConfigFile(fileName string, env common.Environment) (PermissionsMap, error) {
 	jsonFile, err := os.Open(fileName)
 	if err != nil {
 		return nil, fmt.Errorf(`failed to open permissions file "%s": %w`, fileName, err)
@@ -175,21 +182,30 @@ func parseConfigFile(fileName string, allowAnything bool) (PermissionsMap, error
 		return nil, fmt.Errorf(`failed to read permissions file "%s": %w`, fileName, err)
 	}
 
-	retVal, err := parseConfig(byteValue, allowAnything)
+	retVal, err := parseConfig(byteValue, env)
 	if err != nil {
-		return retVal, fmt.Errorf(`failed to parse permissions file "%s": %w`, fileName, err)
+		return nil, fmt.Errorf(`failed to parse permissions file "%s": %w`, fileName, err)
 	}
 
 	return retVal, err
 }
 
 // parseConfig parses the permissions config from a buffer into a map keyed by API key.
-func parseConfig(byteValue []byte, allowAnything bool) (PermissionsMap, error) {
-	var config Config
+func parseConfig(byteValue []byte, env common.Environment) (PermissionsMap, error) {
+	config := Config{DefaultBurstSize: 1}
 	if err := json.Unmarshal(byteValue, &config); err != nil {
 		return nil, fmt.Errorf(`failed to unmarshal json: %w`, err)
 	}
 
+	// According to the docs, a burst size of zero does not allow any events. We don't want that!
+	if config.DefaultBurstSize == 0 {
+		return nil, errors.New("the default burst size may not be zero")
+	}
+
+	if config.AllowAnythingSupported && env == common.MainNet {
+		return nil, fmt.Errorf(`the "allowAnythingSupported" flag is not supported in mainnet`)
+	}
+
 	ret := make(PermissionsMap)
 	userNames := map[string]struct{}{}
 	for _, user := range config.Permissions {
@@ -205,7 +221,7 @@ func parseConfig(byteValue []byte, allowAnything bool) (PermissionsMap, error) {
 		}
 
 		if user.AllowAnything {
-			if !allowAnything {
+			if !config.AllowAnythingSupported {
 				return nil, fmt.Errorf(`UserName "%s" has "allowAnything" specified when the feature is not enabled`, user.UserName)
 			}
 			if len(user.AllowedCalls) != 0 {
@@ -213,6 +229,22 @@ func parseConfig(byteValue []byte, allowAnything bool) (PermissionsMap, error) {
 			}
 		}
 
+		var rateLimiter *rate.Limiter
+		rateLimit := config.DefaultRateLimit
+		if user.RateLimit != nil {
+			rateLimit = *user.RateLimit
+		}
+		if rateLimit != 0 {
+			burstSize := config.DefaultBurstSize
+			if user.BurstSize != nil {
+				burstSize = *user.BurstSize
+			}
+			if burstSize == 0 {
+				return nil, errors.New("if rate limiting is enabled, the burst size may not be zero")
+			}
+			rateLimiter = rate.NewLimiter(rate.Limit(rateLimit), burstSize)
+		}
+
 		// Build the list of allowed calls for this API key.
 		allowedCalls := make(allowedCallsForUser)
 		for _, ac := range user.AllowedCalls {
@@ -312,6 +344,7 @@ func parseConfig(byteValue []byte, allowAnything bool) (PermissionsMap, error) {
 		pe := &permissionEntry{
 			userName:      user.UserName,
 			apiKey:        apiKey,
+			rateLimiter:   rateLimiter,
 			allowUnsigned: user.AllowUnsigned,
 			allowAnything: user.AllowAnything,
 			logResponses:  user.LogResponses,

node/cmd/ccq/query_server.go

@@ -47,7 +47,6 @@ var (
 	shutdownDelay2         *uint
 	monitorPeers           *bool
 	gossipAdvertiseAddress *string
-	allowAnything          *bool
 	verifyPermissions      *bool
 )
 
@@ -71,7 +70,6 @@ func init() {
 	promRemoteURL = QueryServerCmd.Flags().String("promRemoteURL", "", "Prometheus remote write URL (Grafana)")
 	monitorPeers = QueryServerCmd.Flags().Bool("monitorPeers", false, "Should monitor bootstrap peers and attempt to reconnect")
 	gossipAdvertiseAddress = QueryServerCmd.Flags().String("gossipAdvertiseAddress", "", "External IP to advertize on P2P (use if behind a NAT or running in k8s)")
-	allowAnything = QueryServerCmd.Flags().Bool("allowAnything", false, `Should allow API keys with the "allowAnything" flag (only allowed in testnet and devnet)`)
 	verifyPermissions = QueryServerCmd.Flags().Bool("verifyPermissions", false, `parse and verify the permissions file and then exit with 0 if success, 1 if failure`)
 
 	// The default health check monitoring is every five seconds, with a five second timeout, and you have to miss two, for 20 seconds total.
@@ -88,8 +86,18 @@ var QueryServerCmd = &cobra.Command{
 }
 
 func runQueryServer(cmd *cobra.Command, args []string) {
+	env, err := common.ParseEnvironment(*envStr)
+	if err != nil || (env != common.UnsafeDevNet && env != common.TestNet && env != common.MainNet) {
+		if *envStr == "" {
+			fmt.Println("Please specify --env")
+		} else {
+			fmt.Println("Invalid value for --env, should be devnet, testnet or mainnet", zap.String("val", *envStr))
+		}
+		os.Exit(1)
+	}
+
 	if *verifyPermissions {
-		_, err := parseConfigFile(*permFile, *allowAnything)
+		_, err := parseConfigFile(*permFile, env)
 		if err != nil {
 			fmt.Println(err)
 			os.Exit(1)
@@ -109,14 +117,6 @@ func runQueryServer(cmd *cobra.Command, args []string) {
 	logger := ipfslog.Logger("query-server").Desugar()
 	ipfslog.SetAllLoggers(lvl)
 
-	env, err := common.ParseEnvironment(*envStr)
-	if err != nil || (env != common.UnsafeDevNet && env != common.TestNet && env != common.MainNet) {
-		if *envStr == "" {
-			logger.Fatal("Please specify --env")
-		}
-		logger.Fatal("Invalid value for --env, should be devnet, testnet or mainnet", zap.String("val", *envStr))
-	}
-
 	if *p2pNetworkID == "" {
 		*p2pNetworkID = p2p.GetNetworkId(env)
 	} else if env != common.UnsafeDevNet {
@@ -175,14 +175,7 @@ func runQueryServer(cmd *cobra.Command, args []string) {
 		logger.Fatal("Please specify --ethContract")
 	}
 
-	if *allowAnything {
-		if env != common.TestNet && env != common.UnsafeDevNet {
-			logger.Fatal(`The "--allowAnything" flag is only supported in testnet and devnet`)
-		}
-		logger.Info("will allow anything for users for which it is enabled")
-	}
-
-	permissions, err := NewPermissions(*permFile, *allowAnything)
+	permissions, err := NewPermissions(*permFile, env)
 	if err != nil {
 		logger.Fatal("Failed to load permissions file", zap.String("permFile", *permFile), zap.Error(err))
 	}

sdk/js-query/src/query/ethCall.test.ts

@@ -961,4 +961,64 @@ describe("eth call", () => {
       "0x0000000000000000000000000000000000000000000000000000000000000012"
     );
   });
+  test("rate limit exceeded", async () => {
+    const nameCallData = createTestEthCallData(WETH_ADDRESS, "name", "string");
+    const decimalsCallData = createTestEthCallData(
+      WETH_ADDRESS,
+      "decimals",
+      "uint8"
+    );
+    const blockNumber = await web3.eth.getBlockNumber(ETH_DATA_FORMAT);
+    const ethCall = new EthCallQueryRequest(blockNumber, [
+      nameCallData,
+      decimalsCallData,
+    ]);
+    const chainId = 2;
+    for (let bigCount = 0; bigCount < 3; bigCount++) {
+      // We are allowed a burst of two, so these should work.
+      for (let count = 0; count < 2; count++) {
+        const ethQuery = new PerChainQueryRequest(chainId, ethCall);
+        const nonce = count + 1;
+        const request = new QueryRequest(nonce, [ethQuery]);
+        const serialized = request.serialize();
+        const digest = QueryRequest.digest(ENV, serialized);
+        const signature = sign(PRIVATE_KEY, digest);
+        const response = await axios.put(
+          QUERY_URL,
+          {
+            signature,
+            bytes: Buffer.from(serialized).toString("hex"),
+          },
+          { headers: { "X-API-Key": "rate_limited_key" } }
+        );
+        expect(response.status).toBe(200);
+      }
+      // But the next one should fail with a 429.
+      const ethQuery = new PerChainQueryRequest(chainId, ethCall);
+      const nonce = 100;
+      const request = new QueryRequest(nonce, [ethQuery]);
+      const serialized = request.serialize();
+      const digest = QueryRequest.digest(ENV, serialized);
+      const signature = sign(PRIVATE_KEY, digest);
+      let err = false;
+      await axios
+        .put(
+          QUERY_URL,
+          {
+            signature,
+            bytes: Buffer.from(serialized).toString("hex"),
+          },
+          { headers: { "X-API-Key": "rate_limited_key" } }
+        )
+        .catch(function (error) {
+          err = true;
+          expect(error.response.status).toBe(429);
+          expect(error.response.data).toBe("rate limit exceeded\n");
+        });
+      expect(err).toBe(true);
+
+      // But after a sleep, we should be able to go again.
+      await sleep(2000);
+    }
+  });
 });