cloud_functions: tokenAllowlist implementation (#1339)

event_database/cloud_functions/.gitignore

@@ -1,5 +1,6 @@
 vendor
 *.json
+!token-allowlist-*.json
 *.log
 
 *.txt

event_database/cloud_functions/notional-transferred-from.go

@@ -74,10 +74,6 @@ func createTransfersFromOfInterval(tbl *bigtable.Table, ctx context.Context, pre
 
 			// iterate through the rows and increment the amounts
 			for _, row := range queryResult {
-				if _, ok := tokensToSkip[row.TokenAddress]; ok {
-					// skip blacklisted token
-					continue
-				}
 				if _, ok := transfersFromCache.Daily[dateStr][row.LeavingChain]; !ok {
 					transfersFromCache.Daily[dateStr][row.LeavingChain] = 0
 				}

event_database/cloud_functions/notional-transferred-to.go

@@ -46,6 +46,9 @@ type TransferData struct {
 
 // finds all the TokenTransfer rows within the specified period
 func fetchTransferRowsInInterval(tbl *bigtable.Table, ctx context.Context, prefix string, start, end time.Time) []TransferData {
+	if len(tokenAllowlist) == 0 {
+		log.Fatal("tokenAllowlist is empty")
+	}
 	rows := []TransferData{}
 	err := tbl.ReadRows(ctx, bigtable.PrefixRange(prefix), func(row bigtable.Row) bool {
 
@@ -97,7 +100,9 @@ func fetchTransferRowsInInterval(tbl *bigtable.Table, ctx context.Context, prefi
 			keyParts := strings.Split(row.Key(), ":")
 			t.LeavingChain = keyParts[0]
 
-			rows = append(rows, *t)
+			if isTokenAllowed(t.OriginChain, t.TokenAddress) {
+				rows = append(rows, *t)
+			}
 		}
 
 		return true
@@ -193,10 +198,6 @@ func amountsTransferredToInInterval(tbl *bigtable.Table, ctx context.Context, pr
 
 			// iterate through the rows and increment the count
 			for _, row := range queryResult {
-				if _, ok := tokensToSkip[row.TokenAddress]; ok {
-					// skip blacklisted token
-					continue
-				}
 				if _, ok := results[dateStr][row.DestinationChain]; !ok {
 					results[dateStr][row.DestinationChain] = map[string]float64{"*": 0}
 				}
@@ -290,10 +291,6 @@ func transfersToForInterval(tbl *bigtable.Table, ctx context.Context, prefix str
 
 	// iterate through the rows and increment the count for each index
 	for _, row := range queryResults {
-		if _, ok := tokensToSkip[row.TokenAddress]; ok {
-			// skip blacklisted token
-			continue
-		}
 		if _, ok := result[row.DestinationChain]; !ok {
 			result[row.DestinationChain] = map[string]float64{"*": 0}
 		}

event_database/cloud_functions/notional-transferred.go

@@ -98,10 +98,6 @@ func createTransfersOfInterval(tbl *bigtable.Table, ctx context.Context, prefix
 
 			// iterate through the rows and increment the amounts
 			for _, row := range queryResult {
-				if _, ok := tokensToSkip[row.TokenAddress]; ok {
-					// skip blacklisted token
-					continue
-				}
 				if _, ok := results[dateStr][row.LeavingChain]; !ok {
 					results[dateStr][row.LeavingChain] = map[string]map[string]float64{"*": {"*": 0}}
 				}
@@ -254,10 +250,6 @@ func transfersForInterval(tbl *bigtable.Table, ctx context.Context, prefix strin
 
 	// iterate through the rows and increment the count for each index
 	for _, row := range queryResults {
-		if _, ok := tokensToSkip[row.TokenAddress]; ok {
-			// skip blacklisted token
-			continue
-		}
 		if _, ok := result[row.LeavingChain]; !ok {
 			result[row.LeavingChain] = map[string]map[string]float64{"*": {"*": 0}}
 		}

event_database/cloud_functions/notional-tvl-cumulative.go

@@ -264,10 +264,6 @@ func ComputeTvlCumulative(w http.ResponseWriter, r *http.Request) {
 				if symbol == "*" {
 					continue
 				}
-				if _, ok := tokensToSkip[symbol]; ok {
-					log.Printf("going to skip %v, on chain %v, date %v", asset.Symbol, chain, date)
-					continue
-				}
 
 				notional := asset.Amount * asset.TokenPrice
 				if notional <= 0 {

event_database/cloud_functions/notional-tvl.go

@@ -106,10 +106,6 @@ func tvlInInterval(tbl *bigtable.Table, ctx context.Context, start time.Time) ma
 					log.Printf("skipping row without CoinGeckoCoinId. symbol: %v, amount %v", row.TokenSymbol, row.TokenAmount)
 					continue
 				}
-				if _, ok := tokensToSkip[row.TokenAddress]; ok {
-					// skip blacklisted token
-					continue
-				}
 				if row.TokenAddress == "" {
 					// if the token address is missing, skip
 					continue

event_database/cloud_functions/process-transfer.go

@@ -20,7 +20,7 @@ import (
 
 // terra native tokens do not have a bech32 address like cw20s do, handle them manually.
 var tokenAddressExceptions = map[string]string{
-	"0100000000000000000000000000000000000000000000000000000075757364": "ust",
+	"0100000000000000000000000000000000000000000000000000000075757364": "uusd",
 	"010000000000000000000000000000000000000000000000000000756c756e61": "uluna",
 }
 

event_database/cloud_functions/shared.go

@@ -39,38 +39,6 @@ var solanaTokens = map[string]SolanaToken{}
 
 var releaseDay = time.Date(2021, 9, 13, 0, 0, 0, 0, time.UTC)
 
-// token addresses blacklisted from TVL calculation
-var tokensToSkip = map[string]bool{
-	"0x04132bf45511d03a58afd4f1d36a29d229ccc574":   true,
-	"0xa79bd679ce21a2418be9e6f88b2186c9986bbe7d":   true,
-	"0x931c3987040c90b6db09981c7c91ba155d3fa31f":   true,
-	"0x8fb1a59ca2d57b51e5971a85277efe72c4492983":   true,
-	"0xd52d9ba6fcbadb1fe1e3aca52cbb72c4d9bbb4ec":   true,
-	"0x1353c55fd2beebd976d7acc4a7083b0618d94689":   true,
-	"0xf0fbdb8a402ec0fc626db974b8d019c902deb486":   true,
-	"0x1fd4a95f4335cf36cac85730289579c104544328":   true,
-	"0x358aa13c52544eccef6b0add0f801012adad5ee3":   true,
-	"0xbe32b7acd03bcc62f25ebabd169a35e69ef17601":   true,
-	"0x7ffb3d637014488b63fb9858e279385685afc1e2":   true,
-	"0x337dc89ebcc33a337307d58a51888af92cfdc81b":   true,
-	"0x5Cb89Ac06F34f73B1A6b8000CEb0AfBc97d58B6b":   true,
-	"0xd9F0446AedadCf16A12692E02FA26C617FA4D217":   true,
-	"0xD7b41531456b636641F7e867eC77120441D1E1E8":   true,
-	"0x9f607027b69f6e123bc3bd56a686b735fa75f30a":   true,
-	"0x2a35965bbad6fd3964ef815d011c51ab1c546e67":   true,
-	"0x053c070f0923a5b770cc59d7bf74ecff991cd0b8":   true,
-	"0xA18036c8ecb3235087d990c886c242546D1E560f":   true,
-	"0x6B3105826942071E7B6346cbE9867d37Ed7f98Eb":   true,
-	"0x0749902ae8ed9c6a508271bad18f185dba7185d4":   true, // fake WETH on poly
-	"0x4411146b7714f5dc7aa4445fcb44e3ca120c8a1e":   true, // testWETH on poly
-	"0xE389Ac691BD2b0228DAFFfF548fbcE38470373E8":   true, // fake WMATIC on poly
-	"0x7e347498dfef39a88099e3e343140ae17cde260e":   true, // fake wAVAX on bsc
-	"0x685629e5e99e3959254c4d23cd9097fbaef01fb2":   true, // amWeth
-	"terra1vpehfldr2u2m2gw38zaryp4tfw7fe2kw2lryjf": true, // fake btc on terra
-	"0xe9986beb0bcfff418dc4a252904cec370dfb14b8":   true, // fake Dai Stablecoin on bsc
-	"0xdaff96cc3d5e2fa982812ec12ce74833deb51327":   true, // fake btc on bsc
-}
-
 // init runs during cloud function initialization. So, this will only run during an
 // an instance's cold start.
 // https://cloud.google.com/functions/docs/bestpractices/networking#accessing_google_apis
@@ -117,11 +85,10 @@ func init() {
 		cacheBucket = storageClient.Bucket(cacheBucketName)
 	}
 
-	// ensure blacklisted tokens are lowercase
-	for k := range tokensToSkip {
-		tokensToSkip[strings.ToLower(k)] = true
+	tokenAllowlistFilePath := os.Getenv("TOKEN_ALLOWLIST")
+	if tokenAllowlistFilePath != "" {
+		loadJsonToInterface(context.Background(), tokenAllowlistFilePath, &sync.RWMutex{}, &tokenAllowlist)
 	}
-
 }
 
 func timeTrack(start time.Time, name string) {
@@ -479,3 +446,15 @@ func useCache(date string) bool {
 	}
 	return true
 }
+
+// tokens allowed in TVL calculation
+var tokenAllowlist = map[string]map[string]bool{}
+
+func isTokenAllowed(chainId string, tokenAddress string) bool {
+	if tokenAddresses, ok := tokenAllowlist[chainId]; ok {
+		if _, ok := tokenAddresses[tokenAddress]; ok {
+			return true
+		}
+	}
+	return false
+}

event_database/cloud_functions/token-allowlist-devnet.json

@@ -0,0 +1,12 @@
+{
+  "1": {
+    "So11111111111111111111111111111111111111112": true
+  },
+  "2": { "0xddb64fe46a91d46ee29420539fc25fd07c5fea3e": true },
+  "3": {
+    "uluna": true,
+    "uusd": true
+  },
+  "4": { "0xddb64fe46a91d46ee29420539fc25fd07c5fea3e": true },
+  "18": { "uusd": true, "uluna": true }
+}

event_database/cloud_functions/token-allowlist-mainnet.json

@@ -0,0 +1,147 @@
+{
+  "1": {
+    "4k3Dyjzvzp8eMZWUXbBCjEvwSkkk59S5iCNLY3QrkX6R": true,
+    "7dHbWXmci3dT8UFYWYZweBLXgycu7Y3iL6trKn1Y7ARj": true,
+    "AkhdZGVbJXPuQZ53u2LrimCjkRP6ZyxG1SoM85T98eE1": true,
+    "NFTUkR4u7wKxy9QLaX2TGvd9oZSWoMo4jqSJqdMb7Nk": true,
+    "ATLASXmbPQxBUYbxPsV97usA3fPQYEqzQBUHgiFCUsXx": true,
+    "So11111111111111111111111111111111111111112": true,
+    "4Te4KJgjtnZe4aE2zne8G4NPfrPjCwDmaiEx9rKnyDVZ": true,
+    "Es9vMFrzaCERmJfrF4H2FYD4KCoNkY11McCe8BenwNYB": true,
+    "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v": true
+  },
+  "2": {
+    "0xd49efa7bc0d339d74f487959c573d518ba3f8437": true,
+    "0x0c572544a4ee47904d54aaa6a970af96b6f00e1b": true,
+    "0x727f064a78dc734d33eec18d5370aef32ffd46e4": true,
+    "0x0f5d2fb29fb7d3cfee444a200298f468908cc942": true,
+    "0x853d955acef822db058eb8505911ed77f175b99e": true,
+    "0xe28b3b32b6c345a34ff64674606124dd5aceca30": true,
+    "0x4da34f8264cb33a5c9f17081b9ef5ff6091116f4": true,
+    "0x7f39c581f595b53c5cb19bd0b3f8da6c935e2ca0": true,
+    "0x1a7e4e63778b4f12a199c062f3efdd288afcbce8": true,
+    "0x92d6c1e31e14520e676a687f0a93788b716beff5": true,
+    "0x2c537e5624e4af88a7ae4060c022609376c8d0eb": true,
+    "0x0bc529c00c6401aef6d220be8c6ea1667f6ad93e": true,
+    "0x95ad61b0a150d79219dcf64e1e6cc01f0b64c4ce": true,
+    "0x30d20208d987713f46dfd34ef128bb16c404d10f": true,
+    "0x3845badade8e6dff049820680d1f14bd3903a5d0": true,
+    "0x65e6b60ea01668634d68d0513fe814679f925bad": true,
+    "0x8ce9137d39326ad0cd6491fb5cc0cba0e089b6a9": true,
+    "0x72b886d09c117654ab7da13a14d603001de0b777": true,
+    "0x8564653879a18c560e7c0ea0e084c516c62f5653": true,
+    "0x1045f5ccb01daea4f8eab055f5fcbb7c0e7c89f0": true,
+    "0x8a9c67fee641579deba04928c4bc45f66e26343a": true,
+    "0x0316eb71485b0ab14103307bf65a021042c6d380": true,
+    "0x2260fac5e5542a773aa44fbcfedf7c193bc2c599": true,
+    "0xaaaebe6fe48e54f431b0c390cfaf0b017d09d42d": true,
+    "0xbb0e17ef65f82ab018d8edd776e8dd940327b28b": true,
+    "0x05d3606d5c81eb9b7b18530995ec9b29da05faba": true,
+    "0x2ba592f78db6436527729929aaf6c908497cb200": true,
+    "0x41c37a4683d6a05adb31c39d71348a8403b13ca9": true,
+    "0xc00e94cb662c3520282e6f5717214004a7f26888": true,
+    "0x08d967bb0134f2d07f7cfb6e246680c53927dd30": true,
+    "0x111111111117dc0aa78b770fa6a738034120c302": true,
+    "0xef19f4e48830093ce5bc8b3ff7f903a0ae3e9fa1": true,
+    "0x7fc66500c84a76ad7e9c93437bfc5ac33e2ddae9": true,
+    "0x0d8775f648430679a709e98d2b0cb6250d2887ef": true,
+    "0xc944e90c64b2c07662a292be6244bdf05cda44a7": true,
+    "0xdfdb7f72c1f195c5951a234e8db9806eb0635346": true,
+    "0x85eee30c52b0b379b046fb0f85f4f3dc3009afec": true,
+    "0x9b83f827928abdf18cf1f7e67053572b9bceff3a": true,
+    "0xf17e65822b568b3903685a7c9f496cf7656cc6c2": true,
+    "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2": true,
+    "0x3432b6a60d23ca0dfca7761b7ab56459d9c964d0": true,
+    "0x50d1c9771902476076ecfc8b2a83ad6b9355a4c9": true,
+    "0xe831f96a7a1dce1aa2eb760b1e296c6a74caa9d5": true,
+    "0xe0cca86b254005889ac3a81e737f56a14f4a38f5": true,
+    "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48": true,
+    "0xdac17f958d2ee523a2206206994597c13d831ec7": true,
+    "0x5ab6a4f46ce182356b6fa2661ed8ebcafce995ad": true,
+    "0x514910771af9ca656af840dff83e8264ecf986ca": true,
+    "0x6b3595068778dd592e39a122f4f5a5cf09c90fe2": true,
+    "0x707f9118e33a9b8998bea41dd0d46f38bb963fc8": true,
+    "0x1f9840a85d5af5bf1d1762f925bdaddc4201f984": true,
+    "0x2e95cea14dd384429eb3c4331b776c4cfbb6fcd9": true,
+    "0x476c5e26a75bd202a9683ffd34359c0cc15be0ff": true,
+    "0xbba39fd2935d5769116ce38d46a71bde9cf03099": true,
+    "0x18aaa7115705e8be94bffebde57af9bfc265b998": true,
+    "0x4674672bcddda2ea5300f5207e1158185c944bc0": true,
+    "0x27702a26126e0b3702af63ee09ac4d1a084ef628": true,
+    "0x6b175474e89094c44da98b954eedeac495271d0f": true,
+    "0xf1f955016ecbcd7321c7266bccfb96c68ea5e49b": true,
+    "0x5a98fcbea516cf06857215779fd812ca3bef1b32": true,
+    "0xfd09911130e6930bf87f2b0554c44f400bd80d3e": true,
+    "0x4bd70556ae3f8a6ec6c4080a0c327b24325438f3": true,
+    "0x009178997aff09a67d4caccfeb897fb79d036214": true,
+    "0x45804880de22913dafe09f4980848ece6ecbaf78": true
+  },
+  "3": {
+    "terra1hzh9vpxhsk8253se0vv5jj6etdvxu3nv8z07zu": true,
+    "terra1dzhzukyezv0etz22ud940z7adyv7xgcjkahuun": true,
+    "uluna": true,
+    "terra193c42lfwmlkasvcw22l9qqzc5q2dx208tkd7wl": true,
+    "terra13awdgcx40tz5uygkgm79dytez3x87rpg4uhnvu": true,
+    "uusd": true
+  },
+  "4": {
+    "0xe9e7cea3dedca5984780bafc599bd69add087d56": true,
+    "0x74c1815474a75dcb366223107cde1bba4a1a7296": true,
+    "0x8ac76a51cc950d9822d68b83fe1ad97b32cd580d": true,
+    "0xbb4cdb9cbd36b01bd1cbaebf2de08d9173bc095c": true,
+    "0x55d398326f99059ff775485246999027b3197955": true,
+    "0xfafd4cb703b25cb22f43d017e7e0d75febc26743": true,
+    "0x8ebc361536094fd5b4ffb8521e31900614c9f55d": true,
+    "0x7e46d5eb5b7ca573b367275fee94af1945f5b636": true,
+    "0x3019bf2a2ef8040c242c9a4c5c4bd4c81678b2a1": true
+  },
+  "5": {
+    "0x9c891326fd8b1a713974f73bb604677e1e63396d": true,
+    "0x2791bca1f2de4661ed88a30c99a7a9449aa84174": true,
+    "0x7ceb23fd6bc0add59e62ac25578270cff1b9f619": true,
+    "0xc2132d05d31c914a87c6611c10748aeb04b58e8f": true,
+    "0x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270": true
+  },
+  "6": {
+    "0xb31f66aa3c1e785363f0875a1b74e27b85fd66c7": true,
+    "0x4bfc90322dd638f81f034517359bd447f8e0235a": true,
+    "0xa7d7079b0fead91f3e65f86e8915cb59c1a4c664": true,
+    "0x2b2c81e08f1af8835a78bb2a90ae924ace0ea4be": true,
+    "0xb97ef9ef8734c71904d8002f8b6bc66dd9c48a6e": true
+  },
+  "7": {
+    "0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee2": true,
+    "0x3223f17957ba502cbe71401d55a0db26e5f7c68f": true,
+    "0x366ef31c8dc715cbeff5fa54ad106dc9c25c6153": true,
+    "0x21c718c22d52d0f3a789b752d4c2fd5908a8a733": true
+  },
+  "9": {
+    "0x8BEc47865aDe3B172A928df8f990Bc7f2A3b9f79": true,
+    "0x4988a896b1227218e4a686fde5eabdcabd91571f": true,
+    "0xb12bfca5a55806aaf64e99521918a4bf0fc40802": true,
+    "0xc9bdeed33cd01541e1eed10f90519d2c06fe3feb": true,
+    "0xc4bdd27c33ec7daa6fcfd8532ddb524bf4038096": true,
+    "0x5183e1b1091804bc2602586919e6880ac1cf2896": true
+  },
+  "10": {
+    "0x04068da6c83afcfa0e13ba15a6696662335d5b75": true,
+    "0x74b23882a30290451a17c44f4f05243b6b58c76d": true,
+    "0x321162cd933e2be498cd2267a90534a804051b11": true,
+    "0x21be370d5312f44cb42ce377bc9b8a0cef1a4c83": true,
+    "0x260b3e40c714ce8196465ec824cd8bb915081812": true
+  },
+  "11": {
+    "0x0000000000000000000100000000000000000080": true,
+    "0x0000000000000000000100000000000000000081": true
+  },
+  "12": { "0x0000000000000000000100000000000000000000": true },
+  "13": { "0xe4f05a66ec68b54a58b17c22107b02e0232cc817": true },
+  "14": {
+    "0x765de816845861e75a25fca122bb6898b8b1282a": true,
+    "0x471ece3750da237f93b8e339c536989b8978a438": true,
+    "0x46c9757c5497c5b1f2eb73ae79b6b67d119b0b58": true,
+    "0xd8763cba276a3738e6de85b4b3bf5fded6d6ca73": true
+  },
+  "15": { "0xf8ad328e98f85fccbf09e43b16dcbbda7e84beab": true },
+  "18": { "uusd": true, "uluna": true }
+}

event_database/functions_server/.vscode/launch.json

@@ -18,6 +18,7 @@
                 "GCP_PROJECT": "wormhole-315720",
                 "BIGTABLE_INSTANCE": "wormhole-mainnet",
                 "CACHE_BUCKET": "cloud-function-cache-mainnet",
+                "TOKEN_ALLOWLIST": "token-allowlist-mainnet.json",
                 "GOOGLE_APPLICATION_CREDENTIALS": "/home/you/path/to/your/service-account.json",
                 // CoinGecko API key if you have one. will work without - rate limit is lower.
                 // "COINGECKO_API_KEY": "your-key-here",
@@ -50,6 +51,7 @@
                 "PUBSUB_TOKEN_TRANSFER_DETAILS_TOPIC": "create-token-transfer-details-devnet",
                 "PUBSUB_TOKEN_TRANSFER_DETAILS_SUBSCRIPTION": "calculate-transfer-data-devnet",
                 "CACHE_BUCKET": "cloud-function-cache-devnet",
+                "TOKEN_ALLOWLIST": "token-allowlist-devnet.json",
             },
         },
     ]