Validate only tokens file and add dev guide (#112)

* diff only the tokens csv file

* add dev guide

* put both readmes at root

.github/workflows/validate-PR.yml

@@ -24,6 +24,6 @@ jobs:
       - run: yarn install
 
       - name: Validate PR
-        run: yarn dev
+        run: yarn validate-PR
 
 

README-developers.md

@@ -0,0 +1,26 @@
+# Dev Guide to the Token List API 
+Hello curious one -- you found the tech README! 🥷
+
+The [Token List API](https://docs.jup.ag/api/token-list-api) consists of the "Strict" list and "All" list. The Strict list is currently made up of tokens from the original solana token list registry, community validated tokens, and wormhole tokens. We also add metadata tags on all tokens from partner APIs.
+
+Everything that feeds the API can be found in this repo, so you know exactly how it's made and what token data is entering the API.
+
+## Key files
+```bash
+banned-tokens.csv
+validated-tokens.csv 
+src
+   |-- partners
+   |   |-- data
+   |   |-- scripts
+   |-- main.ts
+```
+
+## Tokens and Metadata Tags:
+- `validated-tokens.csv`: Consists original Solana token registry and community validated tokens. Jupiter's API crawls this list, and updates to this file via community PRs will be reflected in the API response.
+- `src/partners/data/wormhole.csv`: Wormhole tokens. An update is crawled via `src/partners/scripts/get-wormhole/ts` every week, and any changes will be made via a PR to this repo.
+- `src/partners/data/solana-fm.csv`: We add metadata of tokens from solana fm that are "verified:true". An update is crawled via `src/partners/scripts/get-solana-fm/ts` every week, and any changes will be made via a PR to this repo.
+
+## API outcome:
+- Our 'All' token list API picks up all of the above tokens and tags
+- Our 'Strict' list filters for tokens from the 'All' list with [these tags](https://docs.jup.ag/api/token-list-api): "wormhole", "original-registry", and "community".

README.md

@@ -28,8 +28,8 @@ Learn more:
 - To validate, leave a comment on the PR like those in [this example](https://github.com/jup-ag/token-list/pull/76). The approval process is held jointly by the team and moderators right now, but attestation from known accounts will show support and speed up the process. Over time, we would like to open it up to community members in a council.
 
 ## Developers: 
-- Feel free to contribute by opening a PR! We would like this to be community owned over time. 
-- Some open issues can be found in the "Issues" tab and we're working on cleaning up our documentation.
+- Feel free to contribute by opening a PR! We would like this to be community owned over time. Check out the "Issues" tab for some ideas on how to contribute!
+- A technical walkthrough of how the inputs feed the Token List API can be found in `README-develoeprs.md`
 - Ideas? Feedback? Comment or talk to us in discord. ❤️
 
 

package.json

@@ -6,7 +6,7 @@
   "author": "raccoons",
   "license": "MIT",
   "scripts": {
-    "dev": "yarn build && node dist/main.js",
+    "validate-PR": "yarn build && node dist/main.js",
     "build": "yarn tsc",
     "update-partners": "yarn build && node dist/partners/scripts/get-wormhole.js && node dist/partners/scripts/get-solana-fm.js",
     "get-wormhole": "yarn build && node dist/partners/scripts/get-wormhole.js",

src/main.ts

@@ -1,17 +1,18 @@
 import * as core from "@actions/core";
 import { exec } from "@actions/exec";
-import { parseGitPatch } from "./parse";
-import { validateGitPatch } from "./validate";
-import { getValidated } from "./get-validated";
+import { parseGitPatch } from "./utils/parse";
+import { validateGitPatch } from "./utils/validate";
+import { getValidated } from "./utils/get-jup-strict";
 import { ValidatedSet, ValidationError } from "./types/types";
 
-async function run(): Promise<void> {
+// Validates diff between validated-tokens.csv in the branch vs origin/main
+async function getDiffAndValidate(): Promise<void> {
 
   let gitDiff = "";
   let gitDiffError = "";
 
   try {
-    await exec("git", ["diff", "origin/main", "-U0", "--color=never"], {
+    await exec("git", ["diff", "origin/main", "validated-tokens.csv"], {
       listeners: {
         stdout: (data: Buffer) => {
           gitDiff += data.toString();
@@ -31,6 +32,7 @@ async function run(): Promise<void> {
 
   // core.debug(`Git diff: ${gitDiff}`)
 
+  // Get Jup tokens that are in the strict list to check for duplicates.
   let validatedSet: ValidatedSet;
   try {
     validatedSet = await getValidated();
@@ -52,4 +54,4 @@ async function run(): Promise<void> {
   }
 }
 
-run();
+getDiffAndValidate();

src/get-validated.ts → src/utils/get-jup-strict.ts

@@ -1,7 +1,7 @@
-import { Token, ValidatedSet } from "./types/types";
+import { Token, ValidatedSet } from "../types/types";
 import fetch from "node-fetch";
 
-/** Read existing validated tokens into Set */
+// Read existing JUP validated tokens into Set
 export async function getValidated(): Promise<ValidatedSet> {
   const names = new Set<string>();
   const symbols = new Set<string>();

src/parse.ts → src/utils/parse.ts

@@ -1,5 +1,6 @@
-import {Patch} from './types/types';
+import {Patch} from '../types/types';
 
+// This is a util function to parse a git diff patch into a more usable format
 export function parseGitPatch(patch: string): Patch[] {
   const lines = patch.split('\n');
 

src/validate.ts → src/utils/validate.ts

@@ -1,28 +1,14 @@
-import { Patch, ValidatedSet, ValidationError } from "./types/types";
+import { Patch, ValidatedSet, ValidationError } from "../types/types";
 import { PublicKey } from "@solana/web3.js";
 
-
-export const VALIDATED_TOKENS_FILE = "validated-tokens.csv";
-
-function isValidatedFile(file: string) {
-  return file === VALIDATED_TOKENS_FILE;
-}
-
+// Validates PR changes to the validated tokens csv file.
+// Checks duplicates to JUP strict list and invalid inputs.
 export function validateGitPatch(patch: Patch, validatedSet: ValidatedSet): ValidationError[] {
   // console.log("Processing patch", patch);
   const errors: ValidationError[] = [];
 
-  // Flag changes to unrelated files
-  if (
-    !isValidatedFile(patch.removed.file) ||
-    !isValidatedFile(patch.added.file)
-  ) {
-    // errors.push(ValidationError.UNRELATED_FILE);
-    // return errors;
-
-    //TODO: Put this back after this PR is merged
-    return []
-  }
+  // TODO: Flag changes to unrelated files
+  // ...
 
   // Flag removals
   if (patch.removed.lines.length > 0) {