Signal AtlasHot Tool Tracker
Admin Login
Back to evidenceBack to tool detailBack to rankingsBack to home

Signal Detail

vgi-tiktoken

This is a raw crawl record from GitHub, currently mapped to #10 on the Newcomers board.

GitHubNEWJun 24, 22:37

Record Summary

Newcomers #10

github+hackernews+rss

Count LLM tokens & chunk text for RAG in DuckDB (Rust, tiktoken) Current movement: NEW. Note: "First time on the board".

Score72
Confidence78%
Surge+9%
Captured atJun 24, 22:37
GitHubapache-arrowduckdb
Current tool mapping
Captured namevgi-tiktoken

This is the name captured in the original signal at crawl time.

Current display namevgi-tiktoken

The system default name is still in use.

Current websitegithub.com

The system default website is still in use.

Current categoryAI Coding

The system default category is still in use.

Open tool research page
Source snapshot
GitHubQuery-farm/vgi-tiktoken
github.com

Count LLM tokens & chunk text for RAG in DuckDB (Rust, tiktoken)

Author / PublisherQuery-farm
Published atJun 24, 22:35
RustStars 0Forks 0
GitHub Repo
Page snapshot
GitHubGitHub - Query-farm/vgi-tiktoken: Count LLM tokens & chunk text for RAG in DuckDB (Rust, tiktoken)
Fetched

Count LLM tokens & chunk text for RAG in DuckDB (Rust, tiktoken) - Query-farm/vgi-tiktoken

Fetched atJun 25, 02:14
Published atWaiting for sync
Domaingithub.com
HTTP status200
AuthorNot recorded
Cache statusFresh cache
This crawl record has already generated a matching evidence card on the tool detail page, so you can jump straight back to it.
Jump to tool evidenceOpen canonical URL
Structured fields
RepositoryQuery-farm/vgi-tiktoken
LanguageRust
Stars0
Forks0
Watchers0
Issues0
Updated At2026-06-24T22:35:56Z
Aggregated Source Count1

Raw Payload

Raw crawl payload

{
  "entity_key": "name:vgitiktoken",
  "source_count": 1,
  "providers": [
    {
      "provider_name": "github",
      "slug": "github-Query-farm-vgi-tiktoken",
      "external_id": "1277573204",
      "external_url": "https://github.com/Query-farm/vgi-tiktoken",
      "metric_value": 225,
      "payload": {
        "id": 1277573204,
        "node_id": "R_kgDOTCY4VA",
        "name": "vgi-tiktoken",
        "full_name": "Query-farm/vgi-tiktoken",
        "private": false,
        "owner": {
          "login": "Query-farm",
          "id": 183420031,
          "node_id": "O_kgDOCu7Efw",
          "avatar_url": "https://avatars.githubusercontent.com/u/183420031?v=4",
          "gravatar_id": "",
          "url": "https://api.github.com/users/Query-farm",
          "html_url": "https://github.com/Query-farm",
          "followers_url": "https://api.github.com/users/Query-farm/followers",
          "following_url": "https://api.github.com/users/Query-farm/following{/other_user}",
          "gists_url": "https://api.github.com/users/Query-farm/gists{/gist_id}",
          "starred_url": "https://api.github.com/users/Query-farm/starred{/owner}{/repo}",
          "subscriptions_url": "https://api.github.com/users/Query-farm/subscriptions",
          "organizations_url": "https://api.github.com/users/Query-farm/orgs",
          "repos_url": "https://api.github.com/users/Query-farm/repos",
          "events_url": "https://api.github.com/users/Query-farm/events{/privacy}",
          "received_events_url": "https://api.github.com/users/Query-farm/received_events",
          "type": "Organization",
          "user_view_type": "public",
          "site_admin": false
        },
        "html_url": "https://github.com/Query-farm/vgi-tiktoken",
        "description": "Count LLM tokens & chunk text for RAG in DuckDB (Rust, tiktoken)",
        "fork": false,
        "url": "https://api.github.com/repos/Query-farm/vgi-tiktoken",
        "forks_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/forks",
        "keys_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/keys{/key_id}",
        "collaborators_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/collaborators{/collaborator}",
        "teams_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/teams",
        "hooks_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/hooks",
        "issue_events_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/issues/events{/number}",
        "events_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/events",
        "assignees_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/assignees{/user}",
        "branches_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/branches{/branch}",
        "tags_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/tags",
        "blobs_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/git/blobs{/sha}",
        "git_tags_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/git/tags{/sha}",
        "git_refs_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/git/refs{/sha}",
        "trees_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/git/trees{/sha}",
        "statuses_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/statuses/{sha}",
        "languages_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/languages",
        "stargazers_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/stargazers",
        "contributors_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/contributors",
        "subscribers_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/subscribers",
        "subscription_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/subscription",
        "commits_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/commits{/sha}",
        "git_commits_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/git/commits{/sha}",
        "comments_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/comments{/number}",
        "issue_comment_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/issues/comments{/number}",
        "contents_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/contents/{+path}",
        "compare_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/compare/{base}...{head}",
        "merges_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/merges",
        "archive_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/{archive_format}{/ref}",
        "downloads_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/downloads",
        "issues_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/issues{/number}",
        "pulls_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/pulls{/number}",
        "milestones_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/milestones{/number}",
        "notifications_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/notifications{?since,all,participating}",
        "labels_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/labels{/name}",
        "releases_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/releases{/id}",
        "deployments_url": "https://api.github.com/repos/Query-farm/vgi-tiktoken/deployments",
        "created_at": "2026-06-23T02:28:28Z",
        "updated_at": "2026-06-24T22:35:56Z",
        "pushed_at": "2026-06-24T22:35:52Z",
        "git_url": "git://github.com/Query-farm/vgi-tiktoken.git",
        "ssh_url": "git@github.com:Query-farm/vgi-tiktoken.git",
        "clone_url": "https://github.com/Query-farm/vgi-tiktoken.git",
        "svn_url": "https://github.com/Query-farm/vgi-tiktoken",
        "homepage": null,
        "size": 165,
        "stargazers_count": 0,
        "watchers_count": 0,
        "language": "Rust",
        "has_issues": true,
        "has_projects": true,
        "has_downloads": true,
        "has_wiki": false,
        "has_pages": false,
        "has_discussions": false,
        "forks_count": 0,
        "mirror_url": null,
        "archived": false,
        "disabled": false,
        "open_issues_count": 0,
        "license": {
          "key": "mit",
          "name": "MIT License",
          "spdx_id": "MIT",
          "url": "https://api.github.com/licenses/mit",
          "node_id": "MDc6TGljZW5zZTEz"
        },
        "allow_forking": true,
        "is_template": false,
        "web_commit_signoff_required": false,
        "has_pull_requests": true,
        "pull_request_creation_policy": "all",
        "topics": [
          "apache-arrow",
          "duckdb",
          "duckdb-extension",
          "llm",
          "query-farm",
          "rag",
          "rust",
          "sql",
          "tiktoken",
          "token-counting",
          "tokenizer",
          "vgi"
        ],
        "visibility": "public",
        "forks": 0,
        "open_issues": 0,
        "watchers": 0,
        "default_branch": "main",
        "temp_clone_token": null,
        "custom_properties": {},
        "organization": {
          "login": "Query-farm",
          "id": 183420031,
          "node_id": "O_kgDOCu7Efw",
          "avatar_url": "https://avatars.githubusercontent.com/u/183420031?v=4",
          "gravatar_id": "",
          "url": "https://api.github.com/users/Query-farm",
          "html_url": "https://github.com/Query-farm",
          "followers_url": "https://api.github.com/users/Query-farm/followers",
          "following_url": "https://api.github.com/users/Query-farm/following{/other_user}",
          "gists_url": "https://api.github.com/users/Query-farm/gists{/gist_id}",
          "starred_url": "https://api.github.com/users/Query-farm/starred{/owner}{/repo}",
          "subscriptions_url": "https://api.github.com/users/Query-farm/subscriptions",
          "organizations_url": "https://api.github.com/users/Query-farm/orgs",
          "repos_url": "https://api.github.com/users/Query-farm/repos",
          "events_url": "https://api.github.com/users/Query-farm/events{/privacy}",
          "received_events_url": "https://api.github.com/users/Query-farm/received_events",
          "type": "Organization",
          "user_view_type": "public",
          "site_admin": false
        },
        "network_count": 0,
        "subscribers_count": 0
      }
    }
  ]
}