瀏覽代碼

Check onchain metadata (#1528)

* WIP newTokensHaveMatchingOnchainMeta() checks decimals against Solana and Name, Symbol with Metaplex metadata (token 2022 not ready yet)

* newTokensHaveMatchingOnchainMeta() works with unittests for findAddedTokens()

* removed dummy data in newTokensHaveMatchingOnchainMeta()

* working Token2022, Token+metaplex fetching. going to refactor for bulk RPC calling and missing token metadata trickledown instead of throwing loud errors

* Token metadata getting works and is robust with unittests
moved to metadata.ts
newTokensHaveMatchinOnchainMeta() untested

* token metadata validation is good to go

* inlined tests were messing up typescript compilation. using vite as a runner should solve this, but for now just move tests to a different file and use vitest for testing and normal TS for running.

newTokensHaveMatchinOnchainMeta() now throws an error if an address is provided that isn't a mint.

* add WHALES as exception for duplicate-symbols check

---------

Co-authored-by: mei <15177990+9yoi@users.noreply.github.com>
Andrew Chiw 1 年之前
父節點
當前提交
f15357840b

+ 5 - 0
.mocharc.json

@@ -0,0 +1,5 @@
+{
+  "require": "ts-node/register",
+  "extension": ["ts"],
+  "watch-files": ["src/**/*.ts"]
+}

+ 11 - 5
package.json

@@ -7,30 +7,36 @@
   "license": "MIT",
   "scripts": {
     "validate-PR": "yarn build && node dist/main.js",
-    "build": "yarn tsc",
+    "build": "tsc",
     "update-partners": "yarn build && node dist/partners/scripts/get-wormhole.js && node dist/partners/scripts/get-solana-fm.js",
     "get-wormhole": "yarn build && node dist/partners/scripts/get-wormhole.js",
     "check-wormhole": "yarn build && node dist/partners/scripts/check-wormhole-with-verified.js",
     "get-solana-fm": "yarn build && node dist/partners/scripts/get-solana-fm.js",
     "format": "yarn prettier --write **/*.ts",
     "format-check": "yarn prettier --check **/*.ts",
-    "lint": "eslint src/**/*.ts"
+    "lint": "eslint src/**/*.ts",
+    "test": "vitest --test-timeout 20000 src/**/*.ts"
   },
   "dependencies": {
     "@actions/core": "^1.10.0",
     "@actions/exec": "^1.1.1",
     "@actions/github": "^5.1.1",
-    "@solana/web3.js": "^1.73.2",
+    "@metaplex-foundation/js": "^0.20.1",
+    "@solana/spl-token": "^0.4.0",
+    "@solana/web3.js": "^1.90.0",
     "@types/minimist": "^1.2.5",
     "csv-writer": "^1.6.0",
     "minimist": "^1.2.8",
     "node-downloader-helper": "^2.1.6",
-    "node-fetch": "^2.6.6"
+    "node-fetch": "^2.6.6",
+    "typescript-node": "^0.1.3"
   },
   "devDependencies": {
     "@types/node": "^18.13.0",
     "@types/node-fetch": "^2.6.2",
     "csv-parse": "^5.5.3",
-    "typescript": "^4.9.5"
+    "ts-node": "^10.9.2",
+    "typescript": "^4.9.5",
+    "vitest": "^1.2.2"
   }
 }

+ 18 - 10
src/logic.ts

@@ -1,7 +1,8 @@
 import * as core from "@actions/core";
 import { exec } from "@actions/exec";
-import { detectDuplicateSymbol, detectDuplicateMints, canOnlyAddOneToken, validMintAddress, noEditsToPreviousLinesAllowed, isCommunityValidated, isSymbolConfusing } from "./utils/validate";
+import { detectDuplicateSymbol, detectDuplicateMints, canOnlyAddOneToken, validMintAddress, noEditsToPreviousLinesAllowed, isCommunityValidated, isSymbolConfusing, newTokensHaveMatchingOnchainMeta, findAddedTokens} from "./utils/validate";
 import { ValidatedTokensData } from "./types/types";
+import { Connection, clusterApiUrl } from "@solana/web3.js";
 import { indexToLineNumber } from "./utils/validate";
 import { parse } from "csv-parse/sync";
 import fs from "fs";
@@ -13,14 +14,15 @@ export async function validateValidatedTokensCsv(filename: string): Promise<numb
     const recordsPreviousRaw = await gitPreviousVersion("validated-tokens.csv");
     fs.writeFileSync(".validated-tokens-0.csv", recordsPreviousRaw);
     const [recordsPrevious, _] = parseCsv(".validated-tokens-0.csv")
-
-    let duplicateSymbols;
-    let duplicateMints;
-    let attemptsToAddMultipleTokens;
-    let invalidMintAddresses;
-    let notCommunityValidated;
-    let noEditsAllowed;
-    let potentiallyConfusingSymbols;
+    const connection = new Connection(clusterApiUrl("mainnet-beta"), "confirmed");
+    let duplicateSymbols = 0;
+    let duplicateMints = 0;
+    let attemptsToAddMultipleTokens = 0;
+    let invalidMintAddresses = 0;
+    let notCommunityValidated = 0;
+    let noEditsAllowed = 0;
+    let potentiallyConfusingSymbols = 0;
+    let doubleCheckMetadataOnChain = 0;
 
     duplicateSymbols = detectDuplicateSymbol(recordsPrevious, records);
     duplicateMints = detectDuplicateMints(records);
@@ -30,6 +32,11 @@ export async function validateValidatedTokensCsv(filename: string): Promise<numb
     notCommunityValidated = isCommunityValidated(records);
     potentiallyConfusingSymbols = isSymbolConfusing(recordsPrevious, records);
 
+    // other validations have their own way of finding newly added tokens. no
+    // time to go through all of them, plus they do something different.
+    const newTokens = findAddedTokens(recordsPrevious, records);
+    doubleCheckMetadataOnChain = await newTokensHaveMatchingOnchainMeta(connection, newTokens);
+
     console.log("No More Duplicate Symbols:", duplicateSymbols, `(${allowedDuplicateSymbols.length} exceptions)`);
     console.log("Duplicate Mints:", duplicateMints);
     console.log("Attempts to Add Multiple Tokens:", attemptsToAddMultipleTokens);
@@ -37,7 +44,8 @@ export async function validateValidatedTokensCsv(filename: string): Promise<numb
     console.log("Not Community Validated:", notCommunityValidated, `(${allowedNotCommunityValidated.length} exceptions)`);
     console.log("Edits to Existing Tokens:", noEditsAllowed);
     console.log("Issues with Symbols in Added Tokens:", potentiallyConfusingSymbols);
-    return (duplicateSymbols + duplicateMints + attemptsToAddMultipleTokens + invalidMintAddresses + noEditsAllowed)
+    console.log("Onchain Metadata Mismatches:", doubleCheckMetadataOnChain);
+    return (duplicateSymbols + duplicateMints + attemptsToAddMultipleTokens + invalidMintAddresses + noEditsAllowed + doubleCheckMetadataOnChain)
 }
 
 // Get previous version of validated-tokens.csv from last commit

+ 1 - 0
src/types/types.ts

@@ -73,6 +73,7 @@ export enum ValidationError {
   INVALID_MINT = "Invalid mint address, not base58 decodable",
   INVALID_DECIMALS = "Invalid decimals",
   INVALID_IMAGE_URL = "Invalid image URL",
+  INVALID_METADATA = "Metadata does not match on-chain data",
   INVALID_COMMUNITY_VALIDATED = "Invalid community validated",
   CHANGES_DISCOURAGED = "Tokens already in the CSV should not be edited"
 }

+ 75 - 0
src/utils/__snapshots__/metadata.spec.ts.snap

@@ -0,0 +1,75 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`should work with a Token/Metaplex-metadata mint 1`] = `
+[
+  [
+    {
+      "decimals": 6,
+      "mint": "JUPyiwrYJFskUPiHa7hkeR8VUtAeFoSYbKedZNsDvCN",
+      "name": "Jupiter",
+      "symbol": "JUP",
+      "uri": "https://static.jup.ag/jup/metadata.json",
+    },
+  ],
+  0,
+]
+`;
+
+exports[`should work with a Token2022 mint 1`] = `
+[
+  [
+    {
+      "decimals": 9,
+      "mint": "HbxiDXQxBKMNJqDsTavQE7LVwrTR36wjV2EaYEqUw6qH",
+      "name": "GH0ST",
+      "symbol": "GH0ST",
+      "uri": "https://bafybeialzeyqbeg7fzhebnknqxancgx3fi7n6xa4uq5lny62sa4xsuouiy.ipfs.dweb.link",
+    },
+  ],
+  0,
+]
+`;
+
+exports[`should work with a Token2022/Community-metadata mint 1`] = `
+[
+  [
+    {
+      "decimals": 5,
+      "mint": "CKfatsPMUf8SkiURsDXs7eK6GWb4Jsd6UDbs7twMCWxo",
+      "name": "BonkEarn",
+      "symbol": "BERN",
+      "uri": "https://api.npoint.io/6276c0cc3ab046e9b770",
+    },
+  ],
+  0,
+]
+`;
+
+exports[`should work with a list of mints 1`] = `
+[
+  [
+    {
+      "decimals": 6,
+      "mint": "JUPyiwrYJFskUPiHa7hkeR8VUtAeFoSYbKedZNsDvCN",
+      "name": "Jupiter",
+      "symbol": "JUP",
+      "uri": "https://static.jup.ag/jup/metadata.json",
+    },
+    {
+      "decimals": 5,
+      "mint": "CKfatsPMUf8SkiURsDXs7eK6GWb4Jsd6UDbs7twMCWxo",
+      "name": "BonkEarn",
+      "symbol": "BERN",
+      "uri": "https://api.npoint.io/6276c0cc3ab046e9b770",
+    },
+    {
+      "decimals": 9,
+      "mint": "HbxiDXQxBKMNJqDsTavQE7LVwrTR36wjV2EaYEqUw6qH",
+      "name": "GH0ST",
+      "symbol": "GH0ST",
+      "uri": "https://bafybeialzeyqbeg7fzhebnknqxancgx3fi7n6xa4uq5lny62sa4xsuouiy.ipfs.dweb.link",
+    },
+  ],
+  0,
+]
+`;

+ 90 - 0
src/utils/__snapshots__/validate.spec.ts.snap

@@ -0,0 +1,90 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`should work with a Token/Metaplex-metadata mint 1`] = `
+[
+  [
+    {
+      "decimals": 6,
+      "mint": "JUPyiwrYJFskUPiHa7hkeR8VUtAeFoSYbKedZNsDvCN",
+      "name": "Jupiter",
+      "symbol": "JUP",
+      "uri": "https://static.jup.ag/jup/metadata.json",
+    },
+  ],
+  0,
+]
+`;
+
+exports[`should work with a Token/Metaplex-metadata mint 2`] = `
+[
+  [
+    {
+      "decimals": 6,
+      "mint": "JUPyiwrYJFskUPiHa7hkeR8VUtAeFoSYbKedZNsDvCN",
+      "name": "Jupiter",
+      "symbol": "JUP",
+      "uri": "https://static.jup.ag/jup/metadata.json",
+    },
+  ],
+  0,
+]
+`;
+
+exports[`should work with a Token2022 mint 1`] = `
+[
+  [
+    {
+      "decimals": 9,
+      "mint": "HbxiDXQxBKMNJqDsTavQE7LVwrTR36wjV2EaYEqUw6qH",
+      "name": "GH0ST",
+      "symbol": "GH0ST",
+      "uri": "https://bafybeialzeyqbeg7fzhebnknqxancgx3fi7n6xa4uq5lny62sa4xsuouiy.ipfs.dweb.link",
+    },
+  ],
+  0,
+]
+`;
+
+exports[`should work with a Token2022/Community-metadata mint 1`] = `
+[
+  [
+    {
+      "decimals": 5,
+      "mint": "CKfatsPMUf8SkiURsDXs7eK6GWb4Jsd6UDbs7twMCWxo",
+      "name": "BonkEarn",
+      "symbol": "BERN",
+      "uri": "https://api.npoint.io/6276c0cc3ab046e9b770",
+    },
+  ],
+  0,
+]
+`;
+
+exports[`should work with a list of mints 1`] = `
+[
+  [
+    {
+      "decimals": 6,
+      "mint": "JUPyiwrYJFskUPiHa7hkeR8VUtAeFoSYbKedZNsDvCN",
+      "name": "Jupiter",
+      "symbol": "JUP",
+      "uri": "https://static.jup.ag/jup/metadata.json",
+    },
+    {
+      "decimals": 5,
+      "mint": "CKfatsPMUf8SkiURsDXs7eK6GWb4Jsd6UDbs7twMCWxo",
+      "name": "BonkEarn",
+      "symbol": "BERN",
+      "uri": "https://api.npoint.io/6276c0cc3ab046e9b770",
+    },
+    {
+      "decimals": 9,
+      "mint": "HbxiDXQxBKMNJqDsTavQE7LVwrTR36wjV2EaYEqUw6qH",
+      "name": "GH0ST",
+      "symbol": "GH0ST",
+      "uri": "https://bafybeialzeyqbeg7fzhebnknqxancgx3fi7n6xa4uq5lny62sa4xsuouiy.ipfs.dweb.link",
+    },
+  ],
+  0,
+]
+`;

+ 27 - 0
src/utils/metadata.spec.ts

@@ -0,0 +1,27 @@
+import { expect, test } from 'vitest';
+import { PublicKey, Connection, clusterApiUrl, AccountInfo } from "@solana/web3.js";
+import { findMetadata } from './metadata';
+
+const JUP = new PublicKey('JUPyiwrYJFskUPiHa7hkeR8VUtAeFoSYbKedZNsDvCN'); // Metaplex metadata
+const BERN = new PublicKey('CKfatsPMUf8SkiURsDXs7eK6GWb4Jsd6UDbs7twMCWxo'); // Community metadata
+const GHOST = new PublicKey('HbxiDXQxBKMNJqDsTavQE7LVwrTR36wjV2EaYEqUw6qH'); // Token2022 Metadata extension
+test('should work with a Token2022 mint', async () => {
+    const connection = new Connection(clusterApiUrl("mainnet-beta"), "confirmed");
+    let t2022Meta = await findMetadata(connection, [GHOST])
+    expect(t2022Meta).toMatchSnapshot()
+})
+test('should work with a Token/Metaplex-metadata mint', async () => {
+    const connection = new Connection(clusterApiUrl("mainnet-beta"), "confirmed");
+    let t2022Meta = await findMetadata(connection, [JUP])
+    expect(t2022Meta).toMatchSnapshot()
+})
+test('should work with a Token2022/Community-metadata mint', async () => {
+    const connection = new Connection(clusterApiUrl("mainnet-beta"), "confirmed");
+    let metadata = await findMetadata(connection, [BERN])
+    expect(metadata).toMatchSnapshot()
+})
+test('should work with a list of mints', async () => {
+    const connection = new Connection(clusterApiUrl("mainnet-beta"), "confirmed");
+    let metadata = await findMetadata(connection, [JUP, BERN, GHOST])
+    expect(metadata).toMatchSnapshot()
+})

+ 137 - 0
src/utils/metadata.ts

@@ -0,0 +1,137 @@
+import { PublicKey, Connection, AccountInfo } from "@solana/web3.js";
+import { unpackMint, Mint, getMetadataPointerState, getExtensionData, ExtensionType } from "@solana/spl-token";
+import { TokenMetadata as T2022Metadata, unpack } from "@solana/spl-token-metadata";
+import { Metadata as MetaplexMetadata, PROGRAM_ID as METAPLEX_METADATA_PROGRAM_ID } from "@metaplex-foundation/mpl-token-metadata";
+
+type MetadataFinder = (connection: Connection, address: PublicKey, accInfo: AccountInfo<Buffer>) => Promise<CommonTokenMetadata | null>;
+
+export async function findMetadata(connection: Connection, addresses: PublicKey[]): Promise<[CommonTokenMetadata[], number]> {
+    let answer: CommonTokenMetadata[] = [];
+    let errors = 0
+    const results = await chunkedGetMultipleAccountInfos(connection, addresses);
+    if (results.length !== addresses.length) {
+        throw new Error(`findMetadata: expected ${addresses.length} results, but got ${results.length}`);
+    }
+
+    // oops, I refactored and findMetaplexMetadata() now needs an additional argument. I'll just use a wrapper function to fix that.
+    const findMetaplexMetadataAdapter: MetadataFinder = async (connection, address, accInfo) => {
+        // Assuming 'metaplex' is the desired default for pdaDerive
+        return findMetaplexMetadata(connection, address, accInfo, 'metaplex');
+    };
+    const findCommunityMetadataAdapter: MetadataFinder = async (connection, address, accInfo) => {
+        // Assuming 'metaplex' is the desired default for pdaDerive
+        return findMetaplexMetadata(connection, address, accInfo, 'community');
+      };
+    for (let [i, result] of results.entries()) {
+        if (result) {
+            const metadata = await findFirstValidMetadata(connection, addresses[i], result, [findToken2022Metadata, findMetaplexMetadataAdapter, findCommunityMetadataAdapter]);
+            if (metadata) {
+                answer.push(metadata)
+            } else {
+                console.log(`could not find on-chain metadata for ${addresses[i]} to doublecheck against`)
+                errors += 1
+            }
+        }
+    }
+
+    return [answer, errors]
+}
+
+async function findFirstValidMetadata(connection: Connection, address: PublicKey, accountInfo: AccountInfo<Buffer>, metadataFinders: MetadataFinder[]) {
+    for (const finder of metadataFinders) {
+        const metadata = await finder(connection, address, accountInfo);
+        if (metadata) {
+            return metadata; // Return the first non-null metadata found
+        }
+    }
+    return null; // Return null if no metadata is found
+}
+
+async function findMetaplexMetadata(connection: Connection, address: PublicKey, accInfo: AccountInfo<Buffer>, pdaDerive: 'metaplex' | 'community'): Promise<CommonTokenMetadata | null> {
+    // You could use getMint(), but it makes an extra RPC call to
+    // getAccountInfo(), which we have to do before anyway (above). So using
+    // unpackMint() saves us one RPC call.
+    const mintInfo = unpackMint(address, accInfo, accInfo.owner);
+
+    const metadataProgramId = pdaDerive === 'metaplex' ? METAPLEX_METADATA_PROGRAM_ID : COMMUNITY_METADATA_PROGRAM_ID;
+    const metadataPda = findMetadataAddress(address, metadataProgramId)
+    try {
+        const metaplexMetadata = await MetaplexMetadata.fromAccountAddress(connection, metadataPda);
+        if (!address.equals(mintInfo.address)) {
+            throw new Error(`findMetaplexMetadata(${address}): sanity check failed: the Mint's address and the address you told me to look up (${address}) should be the same, but they aren't.`)
+        }
+        const answer: CommonTokenMetadata = {
+            mint: mintInfo.address,
+            name: removeEmptyChars(metaplexMetadata.data.name.trim()),
+            decimals: mintInfo.decimals,
+            symbol: removeEmptyChars(metaplexMetadata.data.symbol.trim()),
+            uri: removeEmptyChars(metaplexMetadata.data.uri.trim())
+        }
+        return answer
+    } catch (err) {
+        return null
+    }
+}
+
+async function findToken2022Metadata(connection: Connection, address: PublicKey, accInfo: AccountInfo<Buffer>): Promise<CommonTokenMetadata | null> {
+    // You could use getMint(), but it makes an extra RPC call to
+    // getAccountInfo(), which we have to do before anyway (above). So using
+    // unpackMint() saves us one RPC call.
+    const mintInfo = unpackMint(address, accInfo, accInfo.owner);
+    const metadataPointer = getMetadataPointerState(mintInfo);
+    const metadata = getTokenMetadata(mintInfo);
+    // make sure that the metadata pointer points to the mint account (embedded metadata). Externally hosted metadata is not supported now.
+    if (metadataPointer?.metadataAddress?.equals(address) && metadata && metadata.mint.equals(address)) {
+        let answer: CommonTokenMetadata = {
+            mint: address,
+            name: removeEmptyChars(metadata.name.trim()),
+            decimals: mintInfo.decimals,
+            symbol: removeEmptyChars(metadata.symbol.trim()),
+            uri: removeEmptyChars(metadata.uri.trim())
+        }
+        return answer
+    }
+    // let debug = `error in findToken2022Metadata(${address}), debug info: Metadata pointer should point to mint account: ${metadataPointer?.metadataAddress?.equals(address)}; Metadata should not be null: ${metadata}; Metadata.mint.equals(mint) should be true: ${metadata?.mint.equals(address)}`
+    return null
+}
+
+/* MONOREPO */
+/**
+ * This is not an official program but a community deployement
+ * This was deployed by the fluxbeam team and is controlled by a single signer, to allow token2022 metadata early
+ **/
+const COMMUNITY_METADATA_PROGRAM_ID = new PublicKey('META4s4fSmpkTbZoUsgC1oBnWB31vQcmnN8giPw51Zu');
+const removeEmptyChars = (value: string) => value.replace(/\u0000/g, '');
+function getTokenMetadata(mint: Mint): T2022Metadata | null {
+    const data = getExtensionData(ExtensionType.TokenMetadata, mint.tlvData);
+    if (data === null) {
+        return null;
+    }
+    return unpack(data);
+}
+async function chunkedGetMultipleAccountInfos(
+    connection: Connection,
+    pks: PublicKey[],
+    chunkSize: number = 100,
+) {
+    const chunks = function <T>(array: T[], size: number): T[][] {
+        return Array.apply<number, T[], T[][]>(0, new Array(Math.ceil(array.length / size))).map((_, index) =>
+            array.slice(index * size, (index + 1) * size),
+        );
+    }
+    return (await Promise.all(chunks(pks, chunkSize).map((chunk) => connection.getMultipleAccountsInfo(chunk)))).flat();
+}
+function findMetadataAddress(mint: PublicKey, metadataProgramId: PublicKey): PublicKey {
+    return PublicKey.findProgramAddressSync(
+        [Buffer.from('metadata'), metadataProgramId.toBuffer(), mint.toBuffer()],
+        metadataProgramId,
+    )[0];
+}
+// TokenMetadata is agnostic across Token 2022, Metaplex or Fluxbeam type metadata
+interface CommonTokenMetadata {
+    mint: PublicKey;
+    name: string;
+    decimals: number,
+    symbol: string;
+    uri: string;
+}

+ 94 - 0
src/utils/validate.spec.ts

@@ -0,0 +1,94 @@
+import { Connection, clusterApiUrl } from '@solana/web3.js';
+import { ValidatedTokensData } from '../types/types';
+import { findAddedTokens, newTokensHaveMatchingOnchainMeta } from './validate';
+import { expect, test } from 'vitest'
+
+const kiki: ValidatedTokensData = {
+    Name: "KiKI Token",
+    Symbol: "KIKI",
+    Mint: "DezXAZ8z7PnrnRJjz3wXBoRgixCa6xjnB7YaB1pPB263",
+    Decimals: "9",
+    LogoURI: "https://arweave.net/8mAKLjGGmjKTnmcXeyr3pr7iX13xXVjJJiL6RujDbSPV",
+    Line: 1085,
+    "Community Validated": true,
+}
+const wen: ValidatedTokensData = {
+    Name: "WEN",
+    Symbol: "WEN",
+    Mint: "WENWENvqqNya429ubCdR81ZmD69brwQaaBYY6p3LCpk",
+    Decimals: "5",
+    LogoURI: "https://qgp7lco5ylyitscysc2c7clhpxipw6sexpc2eij7g5rq3pnkcx2q.arweave.net/gZ_1id3C8InIWJC0L4lnfdD7ekS7xaIhPzdjDb2qFfU",
+    Line: 1080,
+    "Community Validated": true,
+}
+const ele: ValidatedTokensData = {
+    Name: "Elementerra",
+    Symbol: "ELE",
+    Mint: "8A9HYfj9WAMgjxARWVCJHAeq9i8vdN9cerBmqUamDj7U",
+    Decimals: "9",
+    LogoURI: "https://elementerra.s3.amazonaws.com/images/elementum.png",
+    Line: 1081,
+    "Community Validated": true,
+}
+
+test('xor() should find the tokens that are not in one list vs the other', () => {
+    const prevTokens = [kiki];
+    const tokens = [kiki, wen, ele];
+    const result = findAddedTokens(prevTokens, tokens);
+    expect(result.length).toBe(2);
+    expect(result[0]).toBe(wen);
+    expect(result[1]).toBe(ele);
+});
+test('xor() does not count tokens missing from the newer list', () => {
+    const prevTokens = [kiki, ele];
+    const tokens = [kiki, wen];
+    const result = findAddedTokens(prevTokens, tokens);
+    expect(result.length).toBe(1);
+    expect(result[0]).toBe(wen);
+});
+test('newTokensHaveMatchingOnchainMeta() works', async () => {
+    const eleL: ValidatedTokensData = {
+        Name: "Elementerra", // onchain says Elementum, so we should have 1 mismatch
+        Symbol: "ELE",
+        Mint: "8A9HYfj9WAMgjxARWVCJHAeq9i8vdN9cerBmqUamDj7U",
+        Decimals: "9",
+        LogoURI: "https://elementerra.s3.amazonaws.com/images/elementum.png", // onchain says JSON file, which we must parse to eventually find this .png file
+        Line: 1081,
+        "Community Validated": true,
+    }
+    const tokens = [eleL];
+    const connection = new Connection(clusterApiUrl("mainnet-beta"));
+    let mismatches = await newTokensHaveMatchingOnchainMeta(connection, tokens);
+    expect(mismatches).toEqual(1);
+
+    mismatches = await newTokensHaveMatchingOnchainMeta(connection, [kiki]); // everything's wrong here because the mint is actually Bonk's
+    expect(mismatches).toEqual(4);
+
+    const jupL: ValidatedTokensData = {
+        Name: "Jupiter",
+        Mint: "JUPyiwrYJFskUPiHa7hkeR8VUtAeFoSYbKedZNsDvCN",
+        Symbol: "JUP",
+        Decimals: "6",
+        LogoURI: "https://static.jup.ag/jup/metadata.json", // if the submitted CSV line has a JSON, and onchain data has a JSON, we should not try to fetch the json.image.
+        Line: 1,
+        "Community Validated": true
+    }
+    mismatches = await newTokensHaveMatchingOnchainMeta(connection, [jupL]);
+    expect(mismatches).toEqual(0);
+});
+test('newTokensHaveMatchingOnchainMeta() errors if the mint doesn\'t exist', async () => {
+    const fake: ValidatedTokensData = {
+        Name: "FAKETOKEN", // onchain says Elementum, so we should have 1 mismatch
+        Symbol: "FAKE",
+        Mint: "6JQq2qS67K4L5xQ3xUTinCyxzdPeZQG1R1ipK8jrY7gc",
+        Decimals: "9",
+        LogoURI: "https://elementerra.s3.amazonaws.com/images/elementum.png", // onchain says JSON file, which we must parse to eventually find this .png file
+        Line: 1,
+        "Community Validated": true,
+    }
+    const tokens = [fake];
+    const connection = new Connection(clusterApiUrl("mainnet-beta"));
+    let mismatches = await newTokensHaveMatchingOnchainMeta(connection, tokens);
+    expect(mismatches).toEqual(1);
+
+});

+ 95 - 8
src/utils/validate.ts

@@ -1,6 +1,7 @@
 import { AllowedException, ValidatedTokensData, ValidationError } from "../types/types";
-import { allowedDuplicateSymbols, allowedNotCommunityValidated} from "./duplicate-symbols";
-import { PublicKey } from "@solana/web3.js";
+import { allowedDuplicateSymbols, allowedNotCommunityValidated } from "./duplicate-symbols";
+import { PublicKey, Connection } from "@solana/web3.js";
+import { findMetadata } from "./metadata";
 
 export function indexToLineNumber(index: number): number {
   return index + 2;
@@ -76,10 +77,10 @@ function xorTokensWithExceptions(tokens: ValidatedTokensData[], allowedDuplicate
   const duplicateSymbolMints = Array.from(setDifference).map((x) => x.split("-"))
   // [['ARB', '9xzZzEHsKnwFL1A3DyFJwj36KnZj3gZ7g4srWp9YTEoh']...]
 
-  const answer : ValidatedTokensData[] = [];
+  const answer: ValidatedTokensData[] = [];
   for (const [symbol, mint] of duplicateSymbolMints) {
     const matchingElement = tokens.find((token) => token.Symbol === symbol && token.Mint === mint);
-    if(matchingElement) {
+    if (matchingElement) {
       answer.push(matchingElement)
     }
   }
@@ -88,7 +89,7 @@ function xorTokensWithExceptions(tokens: ValidatedTokensData[], allowedDuplicate
 
 export function isSymbolConfusing(tokensPreviously: ValidatedTokensData[], tokens: ValidatedTokensData[]): number {
   let problems = 0;
-  const newTokens = xor(tokensPreviously, tokens);
+  const newTokens = findAddedTokens(tokensPreviously, tokens);
 
   // please no more weird symbols. Only alphanumeric, no $/- pre/suffixes, and certainly no emojis either
   const REGEX_NON_ALPHANUMERIC = /[^a-zA-Z0-9]/;
@@ -112,8 +113,10 @@ export function isSymbolConfusing(tokensPreviously: ValidatedTokensData[], token
   return problems
 }
 
-// xor finds the new tokens that were added
-function xor(tokensPreviously: ValidatedTokensData[], tokens: ValidatedTokensData[]):  ValidatedTokensData[] {
+// findAddedTokens returns lines which are in the second list but not in the
+// first list. It does not include lines which were in the first list but aren't
+// in the second.
+export function findAddedTokens(tokensPreviously: ValidatedTokensData[], tokens: ValidatedTokensData[]): ValidatedTokensData[] {
   const answer: ValidatedTokensData[] = [];
   const byMint = new Map();
   tokensPreviously.forEach((token) => {
@@ -205,7 +208,7 @@ export function noEditsToPreviousLinesAllowed(prevTokens: ValidatedTokensData[],
       // the older one didn't. that's completely normal
       if (!areRecordsEqual(prevToken, token)) {
         console.log(ValidationError.CHANGES_DISCOURAGED, prevToken, token)
-        errorCount++;
+        errorCount++
       }
     }
   })
@@ -228,3 +231,87 @@ export function isCommunityValidated(tokens: ValidatedTokensData[]): number {
 
   return errorCount;
 }
+
+export async function newTokensHaveMatchingOnchainMeta(connection: Connection, newTokens: ValidatedTokensData[]): Promise<number> {
+  const mintAddresses = newTokens.map((token) => new PublicKey(token.Mint));
+
+  let [metadatas, errors] = await findMetadata(connection, mintAddresses);
+  if (metadatas.length !== newTokens.length) {
+    console.error(`FATAL ERROR: could not find metadata for one of these tokens (${mintAddresses}). This means there was an account that wasn't a token mint.`)
+    return 1;
+  }
+
+  for (let [i, newToken] of newTokens.entries()) {
+    const metadata = metadatas[i];
+    if (metadata) {
+      // Name mismatch
+      if (metadata.name !== newToken.Name) {
+        console.log(`${ValidationError.INVALID_METADATA}: ${newToken.Mint} Name mismatch Expected: ${newToken.Name}, Found: ${metadata.name}`);
+        errors += 1;
+      }
+
+      // Symbol mismatch
+      if (metadata.symbol !== newToken.Symbol) {
+        console.log(`${ValidationError.INVALID_METADATA}: ${newToken.Mint} Symbol mismatch Expected: ${newToken.Symbol}, Found: ${metadata.symbol}`);
+        errors += 1;
+      }
+
+      // Mint mismatch
+      if (metadata.mint.toString() !== newToken.Mint) {
+        console.log(`${ValidationError.INVALID_METADATA}: ${newToken.Mint} Mint mismatch Expected: ${newToken.Mint}, Found: ${metadata.mint.toString()}`);
+        errors += 1;
+      }
+
+      // URI mismatch
+      // what a mess. On-chain metadata URIs are actually a JSON to a URL
+      // which has the actual Logo URL. So we have to try and fetch before we
+      // make an actual comparison.
+      if (metadata.uri !== newToken.LogoURI) {
+        let uriMismatch = true; // Assume there's a mismatch initially
+        // it might be a JSON. Let's try to fetch it and see if it has an image key
+        if (await checkContentType(metadata.uri) === 'application/json') {
+          const newLogoURI = await getLogoURIFromJson(metadata.uri);
+          if (newLogoURI === newToken.LogoURI) {
+            uriMismatch = false; // The URIs match after fetching the JSON, so no mismatch
+          }
+        }
+
+        if (uriMismatch) {
+          console.log(`${ValidationError.INVALID_METADATA}: ${newToken.Mint} URI mismatch Expected: ${newToken.LogoURI}, Found: ${metadata.uri}`);
+          errors += 1;
+        }
+      }
+
+      // Decimals mismatch
+      if (metadata.decimals !== Number(newToken.Decimals)) {
+        console.log(`${ValidationError.INVALID_METADATA}: ${newToken.Mint} Decimals mismatch Expected: ${newToken.Decimals}, Found: ${metadata.decimals}`);
+        errors += 1;
+      }
+    }
+  }
+  return errors;
+}
+
+type contentType = 'application/json' | 'image' | 'other';
+// checkContentType returns true if the URL points to an image, false if it points to a JSON file
+async function checkContentType(uri: string): Promise<contentType> {
+  const response = await fetch(uri, { method: 'HEAD' });
+  const contentType = response.headers.get('Content-Type');
+  if (contentType === null) {
+    throw new Error(`HTTP HEAD ${uri} failed while checking token.LogoURI`);
+  }
+  if (contentType.startsWith('image/')) {
+    // console.log(`${uri} points to an image.`);
+    return 'image';
+  } else if (contentType === 'application/json') {
+    // console.log(`${uri} points to a JSON file.`);
+    return 'application/json';
+  }
+  return "other"
+}
+
+async function getLogoURIFromJson(uri: string): Promise<string> {
+  const response = await fetch(uri);
+  const json = await response.json();
+  return json.image;
+}

+ 14 - 14
tsconfig.json

@@ -1,17 +1,17 @@
 {
-    "compilerOptions": {
-      "outDir": "./dist",
-      "rootDir": "./src",
-      "target": "ES2015" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
-      "module": "commonjs" /* Specify what module code is generated. */,
-      "resolveJsonModule": true,
-      "esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */,
-      "forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */,
-      "strict": true /* Enable all strict type-checking options. */,
-      "skipLibCheck": true /* Skip type checkng all .d.ts files. */,
-      "allowJs": true,
-      "sourceMap": true
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "target": "ES2015" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
+    "module": "commonjs" /* Specify what module code is generated. */,
+    "resolveJsonModule": true,
+    "esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */,
+    "forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */,
+    "strict": true /* Enable all strict type-checking options. */,
+    "skipLibCheck": true /* Skip type checkng all .d.ts files. */,
+    "allowJs": true,
+    "sourceMap": true,
   },
-  "include": ["src/**/*"],
-  "exclude": ["node_modules"]
+"include": ["src/**/*"],
+"exclude": ["node_modules"]
 }

文件差異過大導致無法顯示
+ 1131 - 24
yarn.lock


部分文件因文件數量過多而無法顯示