"""Render the X tweet-scraper comparison post from data.json + hand-written prose.

Output: site/blog/best-twitter-scraper-apify/index.html (standalone page, site chrome,
design.css). Also copies the harness + analysis scripts into raw/tools/ so the
raw-data section is self-contained. Not wired into build.py on purpose.
"""
import json, html, shutil, datetime as dt
from pathlib import Path

HERE = Path(__file__).resolve().parent
ROOT = HERE.parents[1]
OUTDIR = ROOT / "site/blog/best-twitter-scraper-apify"
RAW = OUTDIR / "raw"
DATA = json.loads((OUTDIR / "data.json").read_text())
R = DATA["runs"]
ORDER = DATA["order"]
CROSS = DATA["cross"]
LOOK = DATA["lookups"]

ASSET_V = "6dd8b56bcc"
DATA_DATE = "2026-09-08"     # census snapshot the six were chosen on (section 1); a re-run does not re-choose
POST_DATE = "2026-09-09"
FIRST_RUN = "2026-09-08"     # first run, kept in full under archive/<date>/
RUN_DATE = "2026-09-19"      # the run every measured number on this page comes from
UPDATED = RUN_DATE
PREV = json.loads((OUTDIR / "archive" / FIRST_RUN / "data.json").read_text())

# ---- census facts used for the selection (snapshot 2026-09-08; 6-week window = 2026-07-28 → 09-08)
SEL = {
    "apidojo/tweet-scraper": dict(short="apidojo / Tweet Scraper V2", why="Widest user base and the oldest listing", users=7459, users_prev=7039, total_users=97945, created="2023-11-24", success=95.4, reviews=195, rating=3.94, listed=0.0004, runs30=4653854, rank="#1 for “tweet scraper”, “twitter scraper” and “twitter” on every day we have"),
    "kaitoeasyapi/twitter-x-data-tweet-scraper-pay-per-result-cheapest": dict(short="kaitoeasyapi / Tweet Scraper (“cheapest”)", why="Second-largest user base; “cheapest” is in its slug", users=4004, users_prev=3767, total_users=23823, created="2024-10-14", success=99.6, reviews=83, rating=4.22, listed=0.00025, runs30=2633111, rank="#2 for “tweet scraper”, #3 for “twitter scraper”"),
    "apidojo/twitter-scraper-lite": dict(short="apidojo / Twitter Scraper Unlimited", why="Highest rating among the high-volume actors", users=1957, users_prev=2123, total_users=33132, created="2024-05-09", success=99.8, reviews=101, rating=4.76, listed=0.0004, runs30=1301762, rank="#2 for “twitter scraper”"),
    "xquik/x-tweet-scraper": dict(short="xquik / X Tweet Scraper", why="Fastest-growing and youngest of the fifteen; lowest listed price", users=1006, users_prev=723, total_users=3626, created="2026-03-28", success=99.7, reviews=14, rating=4.55, listed=0.00015, runs30=1015916, rank="#4 for “tweet scraper” (was #5 on 28 July), #7 for “twitter scraper”"),
    "danek/twitter-scraper": dict(short="danek / Twitter Scraper", why="Most runs of any actor in the set, with a 100.0% 30-day success rate", users=576, users_prev=664, total_users=7737, created="2024-03-28", success=100.0, reviews=20, rating=4.43, listed=0.0003, runs30=8716460, rank="#3 for “tweet scraper”, #6 for “twitter scraper”"),
    "scrape.badger/twitter-tweets-scraper": dict(short="scrape.badger / X Tweet Scraper", why="xquik’s price twin: same listed price, and since 10 August the same title", users=401, users_prev=488, total_users=2673, created="2025-05-17", success=99.6, reviews=17, rating=3.30, listed=0.00015, runs30=1380444, rank="#6 for “tweet scraper”, #9 for “twitter scraper”"),
}
LEFT_OUT = [
    ("api-ninja/x-twitter-advanced-search", "398 users, rated 4.86 from 45 reviews, but $0.015 per tweet: 100× the cheapest actor here. It is a search-only actor and deserves its own test at its own price point."),
    ("scraper_one/x-profile-posts-scraper + x-posts-search", "417 and 313 users. The publisher splits profile posts and search into two actors; you would need both for our three jobs."),
    ("altimis/scweet", "192 users. It entered our candidate list only on the latest snapshot, so we have no six-week history for it."),
    ("automation-lab/twitter-scraper", "145 users, 92.6% success, down from 194 users six weeks ago."),
    ("maximedupre/twitter-scraper", "83 users; its 30-day success rate swung from 50% to 94% inside the six-week window."),
    ("scrapesmith, igolaizola, fastcrawler", "69, 50 and 30 users; the last two run at 87% and 90% success."),
]
# ---- input surface, read from each actor's published input schema (default build; read 2026-09-08, re-read 2026-09-19: same counts, same required fields)
FILTERS = {
    "apidojo/tweet-scraper": dict(fields=26, required="none", modes="URLs, search terms, handles, conversation ids in one run", dates="start / end date fields", lang="yes", engagement="min retweets / likes / replies", media="image, video, quote only", verified="verified, Blue", geo="near, radius, geocode, place id", sort="Top, Latest, both", multi="yes", notes="Clean field names; a custom JavaScript map function for reshaping output."),
    "kaitoeasyapi/twitter-x-data-tweet-scraper-pay-per-result-cheapest": dict(fields=48, required="maxItems", modes="tweet ids, one query string, or a list of search terms", dates="since_time / until_time as unix seconds, or since_id / max_id", lang="yes", engagement="min and max retweets / likes / replies", media="12 filter:* switches (images, videos, spaces, links, news…)", verified="filter:blue_verified", geo="near, within, geocode", sort="Latest, Top, Photos, Videos", multi="yes", notes="Every X search operator is its own field, named after the operator (<code>filter:blue_verified</code>, <code>-min_faves</code>). No profile mode: use <code>from:</code>."),
    "apidojo/twitter-scraper-lite": dict(fields=8, required="none", modes="search terms, handles, URLs", dates="start / end date fields", lang="no", engagement="no", media="no", verified="no", geo="no", sort="Top, Latest, both", multi="yes", notes="The small sibling of Tweet Scraper V2: same output, a fraction of the filters."),
    "xquik/x-tweet-scraper": dict(fields=100, required="none", modes="12 modes (search, profile tweets/replies/media/likes, list, article, replies, quotes…) or auto-route", dates="since / until (hidden fields), plus the operator set", lang="yes", engagement="min and max retweets / likes / replies", media="the same 12 filter:* switches as kaito", verified="filter:blue_verified", geo="near, within, geocode", sort="Latest, Top, both", multi="yes", notes="Roughly a third of the 100 fields are aliases: the item cap alone can be spelled <code>maxItems</code>, <code>maxResults</code>, <code>max_results</code>, <code>resultsLimit</code>, <code>resultsCount</code>, <code>numberOfTweets</code>, <code>maxPosts</code> or <code>max_posts</code>. Output shape is configurable (legacy / rich / raw; camelCase / snake_case; nested / flat)."),
    "danek/twitter-scraper": dict(fields=7, required="max_posts", modes="one username, or one query, or post ids", dates="no field; put operators in the query", lang="no field", engagement="no field", media="search_type Media", verified="no", geo="no", sort="Top, Latest, Media, People, Lists", multi="no: one target per run", notes="Seven flat fields. Which ones apply depends on what you fill in; the schema does not say."),
    "scrape.badger/twitter-tweets-scraper": dict(fields=6, required="mode", modes="8 modes, chosen from a dropdown whose values are UI labels (<code>Get a Few Tweets</code>, <code>Advanced Search</code>)", dates="no field; put operators in the query", lang="no field", engagement="no field", media="query_type Media", verified="no", geo="no", sort="Top, Latest, Media", multi="no: one query per run", notes="No profile mode: use <code>from:</code>. The mode string must match the label exactly."),
}
LIKED = {
    "apidojo/tweet-scraper": (["The richest filter set of the six, with clean names.", "Retweets carry the original post as a nested object.", "Profile job returned the pinned post and flagged it."],
                              ["Long-form posts are cut at roughly 280 characters: 2 of 28 came back whole.", "Second-slowest on the search job (37 s), and the most expensive per tweet.", "Profile output is not sorted; it reads like several fetches concatenated."]),
    "kaitoeasyapi/twitter-x-data-tweet-scraper-pay-per-result-cheapest": (["Full text on long-form posts.", "Charged us $0.00022 per tweet against a listed $0.00025.", "Same record shape as apidojo, so a drop-in swap for many pipelines."],
                              ["Padded a one-tweet lookup with 14 “mock_tweet” rows and billed all 15.", "Returned 60 rows when asked for 50, and billed 60.", "No profile mode: <code>from:apify</code> is a search, so no retweets, and replies dominate. No <code>source</code> field."]),
    "apidojo/twitter-scraper-lite": (["Highest-rated actor in the set (4.76 from 101 reviews).", "Same record shape and same freshness as its big sibling."],
                              ["A $0.05 fee for every single-tweet URL, and $0.016 per search or profile query on top of the per-tweet price.", "Same ~280-character cut on long-form posts.", "Slowest search of the six (40 s), and 51 s to look up one tweet with an image."]),
    "xquik/x-tweet-scraper": (["Fastest on the search (5 s) and profile (6 s) jobs, by a wide margin.", "Full text on long-form posts, and the only actor that flags them (<code>isNoteTweet</code>).", "Profile rows flag retweets and replies, carry the retweeted post in full in the row and attach the original as an object: the best-labelled timeline in the set, and the only one returned in date order.", "Most fields per record (60–62), with media URLs flattened for CSV users."],
                              ["Its search returned a different 100 from the other five: 83 in common, 17 that nobody else returned, 17 that all the others had.", "The only actor here where you also pay platform usage: +5% on our search run.", "100 input fields, many of them aliases."]),
    "danek/twitter-scraper": (["Smallest records (1.6–2.0 KB) and among the fastest.", "Retweets carry the original post in full.", "Charged $0.00028 against a listed $0.0003."],
                              ["Three different record shapes across the three jobs (search, profile, lookup).", "No <code>url</code> field; <code>views</code> is a string.", "Returned 55 rows for a cap of 50, one of them twice, and billed 55."]),
    "scrape.badger/twitter-tweets-scraper": (["Cheapest all-in on every job: $0.14 per 1,000 tweets, and the second-fastest search run (18 s).", "Full text on long-form posts.", "Flat snake_case records, easy to load into a table."],
                              ["No profile mode: <code>from:apify</code> is a search, so no retweets and mostly replies.", "No <code>url</code> field.", "Rated 3.30 from 17 reviews, the lowest in the set."]),
}
CANON_LABELS = [("id", "Tweet id"), ("url", "Tweet URL"), ("text", "Text"), ("created_at", "Created at"), ("lang", "Language"),
                ("author_username", "Author handle"), ("author_name", "Author name"), ("author_id", "Author id"), ("author_followers", "Author followers"),
                ("likes", "Likes"), ("reposts", "Retweets"), ("replies", "Replies"), ("quotes", "Quotes"), ("views", "Views"), ("bookmarks", "Bookmarks"),
                ("is_reply", "Reply flag"), ("is_repost", "Retweet flag"), ("is_quote", "Quote flag"), ("conversation_id", "Conversation id"),
                ("media", "Media"), ("urls", "Links"), ("hashtags", "Hashtags"), ("is_long", "Long-form flag"), ("source", "Source app")]


def e(s):
    return html.escape(str(s))


def short(slug):
    return SEL[slug]["short"]


def store(slug):
    return f"https://apify.com/{slug}"


def _safe(part):
    # mirrors actor_pages._safe — the function that writes the actor pages
    out = "".join(c if (c.isalnum() or c in "-_") else "-" for c in part.lower())
    return out.strip("-") or "actor"


def page(slug):
    u, n = slug.split("/")
    return f"/actors/{_safe(u)}/{_safe(n)}.html"


def usd(v, digits=4):
    return "—" if v is None else f"${v:,.{digits}f}"


def num(v):
    return "—" if v is None else f"{v:,}"


def run(slug, task):
    return R.get(f"{slug}|{task}")


def actor_link(slug, with_slug=False):
    out = f'<a href="{page(slug)}">{e(short(slug))}</a>'
    if with_slug:
        out += f' <span class="mono faint slug">(<a href="{store(slug)}" rel="nofollow">{e(slug)}</a>)</span>'
    return out


def th(cols):
    return "<thead><tr>" + "".join(f"<th>{c}</th>" for c in cols) + "</tr></thead>"


def table(cols, rows, cls="results-table", note=None):
    body = "".join("<tr>" + "".join(f"<td>{c}</td>" for c in r) + "</tr>" for r in rows)
    out = f'<div class="tbl"><table class="{cls}">{th(cols)}<tbody>{body}</tbody></table></div>'
    if note:
        out += f'<p class="table-note">{note}</p>'
    return out


import re as _re


def strip_tags(h):
    return _re.sub(r"<[^>]+>", "", h).replace("&nbsp;", " ")


FAQ = [
    ("Is it legal to scrape Twitter (X)?",
     "We are not lawyers and this is not legal advice. The facts: every tweet in this test is public, and none of the six scrapers asked for an X login. X’s Terms of Service prohibit scraping without permission, which is a contract between X and its account holders. In the United States, <em>hiQ Labs v. LinkedIn</em> (2022) held that scraping publicly available data does not violate the Computer Fraud and Abuse Act. Tweets contain personal data, so storing or processing them can fall under GDPR or CCPA regardless of how they were collected. What you may do with the data depends on where you are and what you do with it; ask counsel for your case."),
    ("Can I scrape tweets without the X API?",
     "Yes. That is what all six actors here do: none asked for an API key, a login or a cookie in the jobs we ran. The trade is that there is no contract with X about rate limits or continuity; the store’s public 30-day success rates for the six ran from 95.4% to 100%."),
    ("How does this compare with X API pricing?",
     "X’s own API is priced per usage: you buy credits and each request deducts from them (docs.x.com, September 2026); the current per-request rates are on X’s pricing page. At the per-tweet prices we were charged, 10,000 tweets cost between $1.40 (scrape.badger) and $4.00 (apidojo), with no credit purchase and no subscription. The X API returns the platform’s own data under its own terms; the scrapers return what a logged-out visitor sees."),
    ("Is there a free Twitter scraper on Apify?",
     "Apify’s free plan includes $5 of usage credit every month, and all six scrapers run on it. At the prices we were charged, $5 buys between 12,500 and 35,000 tweets a month. None of the six is free beyond that credit; a “free” Twitter scraper on the store usually means a free trial or that same platform credit."),
    ("Which Twitter scraper on Apify is the cheapest?",
     "As charged to our account: scrape.badger $0.14 per 1,000 tweets, xquik $0.16 including the platform usage it bills to the user, kaitoeasyapi $0.22, danek $0.28, apidojo $0.40. Watch the minimums: kaitoeasyapi billed 15 rows for a single-tweet lookup, and apidojo’s Twitter Scraper Unlimited charges $0.05 for every single-tweet URL and $0.016 per search or profile query."),
    ("Which scrapers return long-form (note) tweets in full?",
     "kaitoeasyapi, xquik, danek and scrape.badger returned a 1,334-character post whole, in search and on a direct lookup. apidojo’s Tweet Scraper V2 and Twitter Scraper Unlimited returned the first 277 characters. In our 100-tweet search sample, 28 tweets were long-form; apidojo returned two of them whole."),
    ("How do I scrape all tweets from an account?",
     "A timeline actor (apidojo V2, danek, or xquik in profileTweets mode) pages back through the profile the way X shows it; apidojo’s own schema notes that X stops the timeline at roughly 800 posts. For older posts, run a search with date operators (<span class=\"mono\">from:user since:… until:…</span>) in windows; apidojo V2, kaitoeasyapi, xquik and scrape.badger all accept those operators. Sort the output yourself: three of the four timeline actors did not return it in date order; xquik did."),
    ("Which scraper has the most filters?",
     "apidojo’s Tweet Scraper V2, with 26 named fields: date range, language, minimum retweets/likes/replies, image/video/quote filters, verified and Blue filters, geo. kaitoeasyapi and xquik expose X’s full search-operator set as individual fields instead. danek and scrape.badger take a single query string and leave the operators to you."),
    ("Which Twitter scraper is the fastest?",
     "On the 100-tweet search, xquik finished in 5 seconds, scrape.badger in 18, kaitoeasyapi and danek in 20, and the two apidojo actors in 37 and 40. On a single-tweet lookup apidojo V2, xquik and danek answered in 3 to 4 seconds for the image post; the long-form post took 3 seconds (xquik) to 52 (apidojo V2). Eleven days earlier xquik’s search was the slowest of the six at 34 seconds; section 13 lists what moved. All timings include the actor’s cold start."),
]


# ---------------------------------------------------------------- sections
def sec_tldr():
    return f"""
<div class="tldr">
<span class="label">The short answer</span>
<ul>
<li><strong>Cheapest full-text keyword search:</strong> <a href="{page('scrape.badger/twitter-tweets-scraper')}">scrape.badger</a> ($0.14 per 1,000 tweets, 18 s) or <a href="{page('xquik/x-tweet-scraper')}">xquik</a> ($0.16 all-in, 5 s; note that 17 of its 100 results differed from the other five, <a href="#search">section 4</a>).</li>
<li><strong>Profile timeline with retweets, as X shows it:</strong> <a href="{page('apidojo/tweet-scraper')}">apidojo Tweet Scraper V2</a>, <a href="{page('danek/twitter-scraper')}">danek</a> or xquik. Only xquik returned it in date order; sort the other two.</li>
<li><strong>Original posts only, no replies:</strong> none of the six modes we ran does that now; filter on the reply flag (apidojo, kaitoeasyapi, xquik).</li>
<li><strong>Single-tweet lookups:</strong> xquik (3–4 s), danek or scrape.badger (4–7 s). Not apidojo Unlimited ($0.05 each), not kaitoeasyapi (15 billed rows each).</li>
<li><strong>Long-form posts:</strong> any of the four that are not apidojo; both apidojo actors cut at ~280 characters.</li>
<li><strong>Most filters:</strong> apidojo V2. <strong>Smallest records:</strong> danek.</li>
</ul>
<p class="tldr-note">Details and every number behind this list are in <a href="#verdict">section 11</a>; how we tested is in <a href="#method">section 2</a>. This page was re-tested on {RUN_DATE}; <a href="#changes">section 13</a> lists what changed since the first run on {FIRST_RUN}.</p>
</div>"""


def sec_changes():
    """What moved between the first run and this one, from the two data.json files."""
    def old(s, t):
        return PREV["runs"].get(f"{s}|{t}")
    rows = []
    for s in ORDER:
        o, n = old(s, "search"), run(s, "search")
        op, np_ = old(s, "profile"), run(s, "profile")
        om, nm = old(s, "lookup_media"), run(s, "lookup_media")
        on, nn = old(s, "lookup_note"), run(s, "lookup_note")
        tot_o = sum((old(s, t) or {}).get("usd_total") or 0 for t in ("search", "profile", "lookup_media", "lookup_note"))
        tot_n = sum((run(s, t) or {}).get("usd_total") or 0 for t in ("search", "profile", "lookup_media", "lookup_note"))
        arrow = lambda a, b, f="{:.1f} s": f'<span class="mono">{f.format(a)} → {f.format(b)}</span>'
        rows.append([actor_link(s), f'<span class="mono">{e(o["build"])} → {e(n["build"])}</span>', arrow(o["duration_s"], n["duration_s"]), arrow(op["duration_s"], np_["duration_s"]),
                     f'<span class="mono">{om["duration_s"]:.0f} / {on["duration_s"]:.0f} s → {nm["duration_s"]:.0f} / {nn["duration_s"]:.0f} s</span>',
                     f'<span class="mono">{op["items"]["n"]} → {np_["items"]["n"]}</span>', f'<span class="mono">{usd(tot_o, 4)} → {usd(tot_n, 4)}</span>'])
    ps, pn = PREV["cross"]["search"], CROSS["search"]
    px, nx = PREV["runs"]["xquik/x-tweet-scraper|search"], run("xquik/x-tweet-scraper", "search")
    prev_pct = px["platform_usage_usd"] / (px["usd_total"] - px["platform_usage_usd"]) * 100
    cur_pct = nx["platform_usage_usd"] / (nx["usd_total"] - nx["platform_usage_usd"]) * 100
    pp, pq = PREV["cross"]["profile"], CROSS["profile"]
    return f"""
<section class="post-section" id="changes">
<h2>13. What changed between 8 and 19 September</h2>
<p>Same six actors, same plan, same inputs, same scripts, same account, eleven days apart. The first run is kept in full under <a href="archive/{FIRST_RUN}/">archive/{FIRST_RUN}/</a>. The table is computed from the two <span class="mono">data.json</span> files; the list below it is our reading.</p>
{table(["Actor", "Build", "Search time", "Profile time", "Lookups (image / long-form)", "Profile rows", "Charged, 4 runs"], rows, cls="results-table wide", note="Left of each arrow: 8 September. Right: 19 September. Build is the actor build number the platform recorded on the search run.")}
<ul class="plain">
<li><strong>xquik’s build number advanced by 73 (1.12.144 → 1.12.217), and two things we measure changed.</strong> Its search went from the slowest of the six (34 s) to the fastest (5 s), and stopped returning the same 100 tweets as the other five: 83 shared, 17 only it returned, 17 it left out (section 4). Its <span class="mono">profileTweets</span> mode went from “original posts and retweets, no replies, retweets as unflagged stubs” to the timeline as X shows it: 47 of 50 in common with apidojo V2, retweets and replies flagged, the retweeted text in full, the original attached, in date order (section 5). “Retweets are not flagged and the original post is not included” was one of our four “did not like” items for xquik on 8 September; it is gone, and so is “slowest on the search job”. Platform usage billed to the user fell from {prev_pct:.0f}% to {cur_pct:.0f}% of the search bill because the run is shorter.</li>
<li><strong>apidojo’s build numbers advanced by 11 on each actor.</strong> Search got slower (25 → 37 s and 24 → 40 s). Twitter Scraper Unlimited returned and flagged the pinned post this time, which it had dropped on 8 September. The long-form cut did not move: 2 of 28 whole in search, 277 of 1,334 characters on the direct lookup, on both dates.</li>
<li><strong>kaitoeasyapi</strong> (build number +3): the same 14 <span class="mono">mock_tweet</span> filler rows on a one-tweet lookup, 60 rows for a cap of 50, billed 60. Unchanged.</li>
<li><strong>danek</strong> (no new build): 55 rows for a cap of 50 where it had returned 50 with two duplicates; billed 55. Otherwise the same.</li>
<li><strong>scrape.badger</strong> (no new build): the same results, 3 seconds slower on search (15 → 18 s) and no longer the fastest.</li>
<li><strong>Retrieval agreement.</strong> Search: {ps["in_all"]} of {ps["union"]} ids in all six on 8 September; {pn["in_all"]} of {pn["union"]} on 19 September, all of the difference from xquik. Profile: {pp["in_all"]} of {pp["union"]} in all six, then {pq["in_all"]} of {pq["union"]}; the six agree more because xquik’s mode now returns the timeline.</li>
<li><strong>What did not change:</strong> the cheapest and most expensive per 1,000 (scrape.badger $0.14, apidojo $0.40), the applied-versus-listed prices, apidojo Unlimited’s $0.05 single-tweet fee, the field coverage table except xquik’s new retweet flag, and the input schemas.</li>
<li><strong>Method.</strong> One change to our scripts: <span class="mono">mapping.py</span> now reads xquik’s <span class="mono">isRetweet</span> flag, which did not exist on 8 September. Everything else ran unchanged; the copy under <span class="mono">raw/tools/</span> is what ran.</li>
</ul>
</section>"""


def sec_faq():
    items = "".join(f"<details class=\"faq\" open><summary><h3>{e(q)}</h3></summary><p>{a}</p></details>" for q, a in FAQ)
    return f"""
<section class="post-section" id="faq">
<h2>14. Questions people ask about scraping Twitter (X)</h2>
<p>Short answers from this test, for the questions that come up most in search.</p>
{items}
</section>"""


def sec_time():
    links = ", ".join(f'<a href="{page(s)}">{e(short(s))}</a>' for s in ORDER)
    return f"""
<div class="time-note">
<span class="label">A note on time</span>
<p>Every measured number on this page comes from runs made on 19 September 2026. The first run, on 8 September, is kept in full under <a href="archive/2026-09-08/">archive/2026-09-08/</a>, and <a href="#changes">section 13</a> lists what moved between the two. Between the runs the build number advanced by 73 on xquik, by 11 on each apidojo actor, by 3 on kaitoeasyapi, and not at all on danek and scrape.badger. By the time you read this, some of it will be stale and some of it may simply be wrong. These actors change often, and a few of them change daily: in the six weeks before the first test, xquik renamed its listing twice, apidojo renamed Tweet Scraper V2 once, scrape.badger and xquik each changed their pricing once, apidojo’s two actors shipped a new build on every one of the 18 days we looked, xquik shipped 12, kaitoeasyapi 5, scrape.badger 2, and danek did not change once. We record every listing in the store every day. If a number here matters to you, open the actor’s page on this site and follow it; you will see the change the day it happens. The six: {links}.</p>
</div>"""


def sec_selection():
    rows = []
    for s in ORDER:
        f = SEL[s]
        g = (f["users"] - f["users_prev"]) / f["users_prev"] * 100
        gcls = "up" if g > 0 else "down"
        rows.append([actor_link(s, True) + f'<span class="why">{e(f["why"])}</span>', f'<span class="mono">{f["users"]:,}</span>',
                     f'<span class="mono {gcls}">{g:+.0f}%</span>', f'<span class="mono">{f["created"]}</span>',
                     f'<span class="mono">{f["success"]:.1f}%</span>', f'<span class="mono">{f["rating"]:.2f}</span> <span class="faint">/ {f["reviews"]}</span>',
                     f'<span class="mono">${f["listed"]:.5f}</span>'])
    t = table(["Actor and why it is in", "Users, 30 d", "6-week change", "First published", "Success, 30 d", "Rating / reviews", "Listed price"], rows,
              note=f"Census snapshot of {DATA_DATE}. “6-week change” compares monthly users on 28 July and 8 September. “First published” is the actor’s creation date from the Apify API. Success is the store’s public 30-day run statistic. Listed price is the primary pay-per-event price shown on the store listing.")
    left = "".join(f"<li><strong>{e(a)}</strong> — {b}</li>" for a, b in LEFT_OUT)
    return f"""
<section class="post-section" id="selection">
<h2>1. How we chose the six tweet scrapers</h2>
<p>The census behind this site tracks <a href="/platforms/x-com.html">every X (Twitter) actor on the Apify Store</a>; 53 of them have at least 30 monthly users. We removed everything that is not a tweet scraper: profile and user scrapers, follower scrapers, trend scrapers, list scrapers, reply-only scrapers, a video downloader. Fifteen remained.</p>
<p>Ranking the fifteen by monthly users and taking the top six would put four actors from two publishers on the page and tell you little. We took one actor per reason instead. The six were chosen on 8 September from that day’s census and were not re-chosen for the re-run; the table below is the census as it stood then.</p>
{t}
<h3>Two of the six share a name</h3>
<p>Since 10 August the listings of <em>xquik/x-tweet-scraper</em> and <em>scrape.badger/twitter-tweets-scraper</em> carry the same title, character for character: <span class="mono">X Tweet Scraper | $0.15/1K Tweets | Pay-Per-Result</span>. scrape.badger’s is the older listing (May 2025 against March 2026). Our title history starts on 24 July, so we cannot say which wording came first. On 24 July xquik’s title still read <span class="mono">From $0.15/1K Tweets | Pay-Per Result</span>; it changed on 31 July and again on 10 August and arrived at the identical string. We report this because a buyer scanning the store sees two identical names and has no way to tell them apart. The rest of this page is one way.</p>
<h3>Who we left out, and why</h3>
<ul class="plain">{left}</ul>
</section>"""


def inputs_block(task):
    parts = []
    for s in ORDER:
        r = run(s, task)
        if not r:
            continue
        parts.append(f"<div class=\"inp\"><div class=\"inp-name\">{e(short(s))}</div><pre><code>{e(json.dumps(r['input'], indent=1))}</code></pre></div>")
    return '<div class="inputs">' + "".join(parts) + "</div>"


def sec_method():
    canon = ", ".join(l for _, l in CANON_LABELS)
    return f"""
<section class="post-section" id="method">
<h2>2. What we asked each scraper to do</h2>
<p>Three jobs, the same for everyone.</p>
<ol class="jobs">
<li><strong>Keyword search.</strong> The phrase <span class="mono">web scraping</span>, newest first, 100 tweets.</li>
<li><strong>Profile timeline.</strong> The 50 newest posts by <span class="mono">@apify</span>.</li>
<li><strong>Single-tweet lookup.</strong> Two tweets by id: <a href="https://x.com/apify/status/2095479050911309827" rel="nofollow">a post with an image and some engagement</a>, and <a href="https://x.com/i/status/2096838373524812149" rel="nofollow">a long-form post of 1,334 characters</a>.</li>
</ol>
<h3>Rules</h3>
<ul class="plain">
<li>We set only the query, the sort (Latest), the item cap and the handle or id. Everything else stayed at the actor’s documented default or console prefill. Where an actor’s console prefills an output shape (xquik: rich, camelCase, flat), we kept the prefill.</li>
<li>Two actors have no profile mode. kaitoeasyapi and scrape.badger ran the profile job as a search for <span class="mono">from:apify</span>, which is what their READMEs suggest. Section 5 shows what that does to the result.</li>
<li>All 24 runs were started inside the same second (12:56:49 UTC, 19 September 2026) from one Apify account on the Starter plan, through the API, with each actor’s default memory. On the first run (8 September) the twelve search and profile runs started at 22:13:48 UTC and the lookups about ten minutes later.</li>
<li>One run per job per actor. No retries. We kept whatever came back.</li>
<li>Costs are read from the run object after charges settled. Durations are the platform’s own <span class="mono">startedAt</span> to <span class="mono">finishedAt</span>, so they include the actor’s cold start.</li>
</ul>
<h3>What we expected back</h3>
<p>A tweet record with, at minimum: {e(canon)}. Section 7 shows who returns which.</p>
<details><summary>Exact inputs, keyword search (6)</summary>{inputs_block('search')}</details>
<details><summary>Exact inputs, profile timeline (6)</summary>{inputs_block('profile')}</details>
<details><summary>Exact inputs, single-tweet lookups (12)</summary>{inputs_block('lookup_media')}{inputs_block('lookup_note')}</details>
</section>"""


def glance_table(task, label):
    lookup = task.startswith("lookup")
    rows = []
    for s in ORDER:
        r = run(s, task)
        if not r:
            continue
        it = r["items"]
        n = it["n"]
        real = it.get("real_items", n)
        ncell = f'<span class="mono">{n}</span>' + (f' <span class="bad-text">({it["filler"]} filler)</span>' if it.get("filler") else "")
        billing = "user pays" if r["billing_model"] == "USER" else "publisher pays"
        row = [actor_link(s), ncell, f'<span class="mono">{r["duration_s"]:.1f} s</span>', f'<span class="mono">{usd(r["usd_total"])}</span>']
        if not lookup:
            row.append(f'<span class="mono">{usd(r["usd_per_1k"], 3)}</span>')
        row += [f'<span class="mono">${r["listed_price_per_item"]:.5f}</span>', billing,
                f'<span class="mono">{r["bytes_per_item"]:,}</span>', f'<span class="mono">{r["keys_per_item"]}</span>']
        rows.append(row)
    cols = ["Actor", "Rows", "Time", "Charged"] + ([] if lookup else ["Per 1,000 rows"]) + ["Event price applied", "Platform usage", "Bytes / row", "Fields / row"]
    return f"<h3>{label}</h3>" + table(cols, rows)


def sec_glance():
    total = sum(v["usd_total"] or 0 for v in R.values())
    return f"""
<section class="post-section" id="glance">
<h2>3. Results at a glance</h2>
<p>Every run finished with status SUCCEEDED. Total charged for all 24 runs: <span class="mono">{usd(total, 2)}</span>.</p>
{glance_table('search', 'Keyword search: 100 tweets for “web scraping”')}
{glance_table('profile', 'Profile timeline: 50 newest posts by @apify')}
{glance_table('lookup_media', 'Single tweet, the one with an image')}
{glance_table('lookup_note', 'Single tweet, the long-form one')}
<p class="table-note">“Charged per row” is the per-event price the platform applied to our run, which is not always the listed price (section 9). “Platform usage” says who pays compute and storage for the run: for five actors the publisher absorbs it, for xquik the user pays it on top of the per-tweet price. “Fields / row” is the average number of top-level keys per record.</p>
</section>"""


def sec_search():
    x = CROSS["search"]
    lf = x["longform_capture"]
    rows = []
    for s in ORDER:
        r = run(s, "search")
        it = r["items"]
        l = lf[s]
        rows.append([actor_link(s), f'<span class="mono">{l["full_text"]} / {l["long_ids_seen"]}</span>', f'<span class="mono">{it["text_max"]:,}</span>',
                     f'<span class="mono">{r["duration_s"]:.1f} s</span>', f'<span class="mono">{usd(r["usd_per_1k"], 3)}</span>'])
    ma = x["metric_agreement"]
    shared = x["matrix"]["xquik/x-tweet-scraper"]["danek/twitter-scraper"]
    return f"""
<section class="post-section" id="search">
<h2>4. Scraping tweets by keyword: the same 100 tweets for five of the six, different text and different bills</h2>
<p>Five actors returned the same 100 tweets. apidojo V2, apidojo Unlimited, kaitoeasyapi, danek and scrape.badger agree on every id, and the newest tweet is the same across all six (12:53:34 UTC, three minutes before the runs). Whatever these five do behind the scenes, they read the same source at the same moment. xquik did not: it shares {shared} ids with the other five, returned 17 that none of them did, and left out 17 that all of them had. The union of the six result sets is {x["union"]} ids and {x["in_all"]} appear in all six. Of the 17 only xquik returned, 4 contain the phrase “web scraping”; of the 17 it left out, 9 do. On the first run, eleven days earlier, xquik agreed with the rest (99 of 101 ids in all six) and its search took 34 seconds; this time it took 5, and it did not read the same result set as the other five. View counts are identical for {ma["views"]["identical"]} of {ma["views"]["ids"]} shared tweets.</p>
<p>Where the five differ is the text. X allows posts far beyond 280 characters (long-form, or “note” tweets). 28 of the 100 tweets the five agreeing actors returned are long-form. kaitoeasyapi, danek and scrape.badger return 27 of them whole, xquik 34 of the 35 it saw. apidojo’s two actors return the first ~280 characters and stop, on 26 of the 28.</p>
{table(["Actor", "Long-form posts returned in full", "Longest text, chars", "Time", "Per 1,000"], rows, note="“In full” means the text is at least 95% as long as the longest version any actor returned for the same id. The long-form set is the ids xquik flags as isNoteTweet; the other actors do not flag them, but four of them return the same 1,000–6,000-character bodies. The one long-form post that no actor but apidojo returns “in full” is a 283-character post that apidojo returns with an expanded link, at 307 characters; the others’ 283 fall under the 95% rule, not under a cut. xquik sees 35 because 7 of its 17 extra tweets are long-form.")}
<p>Speed ran from 5 s (xquik) to 40 s (apidojo Unlimited) for 100 tweets. Price per 1,000 ran from $0.14 (scrape.badger) to $0.40 (both apidojo actors). Records ran from 2.0 KB (danek) to 7.7 KB (xquik) each; the difference is nested author objects, entity lists and, in xquik’s case, about thirty flattened <span class="mono">author*</span> fields per row.</p>
</section>"""


def sec_profile():
    x = CROSS["profile"]
    base = "apidojo/tweet-scraper"
    rows = []
    for s in ORDER:
        r = run(s, "profile")
        it = r["items"]
        replies = it["reply_flagged"] if it["reply_flagged"] is not None else f'{it["reply_by_text"]}*'
        rt_note = ""
        if it["repost_by_text"]:
            rt_note = f' <span class="faint">({it["repost_truncated"]} truncated in the row, original attached for {it["repost_with_original"]})</span>'
        rows.append([actor_link(s), f'<span class="mono">{it["n"]}</span>' + (f' <span class="bad-text">({it["dups"]} dup)</span>' if it["dups"] else ""),
                     f'<span class="mono">{it["repost_by_text"]}</span>{rt_note}', f'<span class="mono">{replies}</span>',
                     f'<span class="mono">{it["newest"][:16].replace("T", " ")}</span>', f'<span class="mono">{it["oldest"][:10]}</span>',
                     f'<span class="mono">{it["newest_first_pct"]:.0f}%</span>', f'<span class="mono">{x["matrix"][s][base]}</span>',
                     f'<span class="mono">{r["duration_s"]:.1f} s</span>'])
    return f"""
<section class="post-section" id="profile">
<h2>5. Profile timeline: “the 50 newest posts by @apify” still means different things</h2>
<p>This is the job where the six stop agreeing. The union of the six outputs is {x["union"]} distinct posts; {x["in_all"]} of them appear in all six, and {x["only_one"]} appear in exactly one. </p>
{table(["Actor", "Rows", "Retweets", "Replies", "Newest post (UTC)", "Oldest post", "Sorted newest-first", "Overlap with apidojo V2", "Time"], rows, note="Retweets are rows whose text starts with “RT @”. Replies use the actor’s own flag; * marks actors without a reply flag, where we counted texts that start with “@”. “Sorted newest-first” is the share of consecutive rows in descending date order. Overlap is the number of ids in common with apidojo/tweet-scraper’s 50.")}
<ul class="plain">
<li><strong>apidojo V2 and Unlimited</strong> return the account’s timeline as X shows it: original posts, 11 retweets and 14 replies, with the pinned post first and flagged by both. The rows are not in date order: the pinned post, then a block that descends from 18 September to 15 September, then a jump back to 19 September and a second descent, and so on, which looks like several fetches concatenated. Sort before you use it. The retweet rows carry a cut text (“RT @…”, 10 of 11 end in “…”) with the original post attached as an object.</li>
<li><strong>xquik</strong> in <span class="mono">profileTweets</span> mode now returns the timeline too: 47 of its 50 are in apidojo V2’s 50, with 10 retweets and 16 replies, every one flagged (<span class="mono">isRetweet</span>, <span class="mono">isReply</span>), the retweeted text in full in the row and the original attached as <span class="mono">retweetedTweet</span>, in strict date order, in 6 seconds. (The same input returned no replies on the first run; <a href="#changes">section 13</a>.)</li>
<li><strong>danek</strong> returns nearly the same set as apidojo (49 of 50 in common), attaches the original post to every retweet, and finished in 9 seconds. It returned 55 rows for a cap of 50, one of them twice, and billed 55.</li>
<li><strong>kaitoeasyapi and scrape.badger</strong> have no profile mode, so <span class="mono">from:apify</span> is a search. Search excludes retweets and includes every reply, so their “timeline” is 29 and 28 replies out of 60 and 50 rows, reaches back only to 5 and 9 September, and has 32 and 31 posts in common with apidojo V2. kaitoeasyapi again returned 60 rows for a cap of 50 and billed 60; its schema warns that “the final response may slightly exceed the specified max_items”.</li>
</ul>
<p>None of these is wrong. They are different definitions of a profile pull. If you need what a visitor sees on the profile page, use a timeline actor; xquik gives it to you sorted and labelled, apidojo V2 and danek need a sort. If you need only original posts, no mode we ran does that now: filter on the reply and retweet flags. If you need replies, the two search-based actors give you mostly that.</p>
</section>"""


def sec_lookups():
    rows = []
    for s in ORDER:
        m, n = LOOK["lookup_media"][s], LOOK["lookup_note"][s]
        rm, rn = run(s, "lookup_media"), run(s, "lookup_note")
        fill = f' <span class="bad-text">+{m["filler"]} filler rows</span>' if m and m.get("filler") else ""
        rows.append([actor_link(s), f'<span class="mono">{m["text_len"]}</span>{fill}', f'<span class="mono">{n["text_len"]:,}</span>',
                     f'<span class="mono">{num(m["likes"])} / {num(m["views"])}</span>', f'<span class="mono">{rm["duration_s"]:.1f} s / {rn["duration_s"]:.1f} s</span>',
                     f'<span class="mono">{usd(rm["usd_total"])} / {usd(rn["usd_total"])}</span>'])
    mock = "From KaitoEasyAPI, a reminder: Our API pricing is based on the volume of data returned. However, to ensure we can cover our costs on the Apify platform, we have a minimum charge of $X per API call, even if the response contains no results. Thus, we returned N pieces of mock data. We will monitor and adjust the size of N based on the infrastructure costs incurred by Apify."
    return f"""
<section class="post-section" id="lookup">
<h2>6. Single-tweet lookup: one id, six answers</h2>
{table(["Actor", "Image post: text, chars", "Long-form post: text, chars", "Likes / views (image post)", "Time (image / long-form)", "Charged (image / long-form)"], rows)}
<ul class="plain">
<li><strong>The long-form post is 1,334 characters.</strong> Four actors returned all of it. apidojo V2 and Unlimited returned 277 characters, the same cut as in the search job. This is a direct lookup of one id, so it is not a paging artefact; the actors do not read the long-form body.</li>
<li><strong>kaitoeasyapi returned 15 rows for one tweet and billed 15.</strong> One row is the tweet. The other fourteen are of type <span class="mono">mock_tweet</span> with id −1 and this text: <q>{e(mock)}</q> The publisher states the policy in the row itself; the store listing says $0.00025 per tweet and the input schema says nothing about a minimum. On a one-tweet lookup the effective price was $0.0033, fifteen times the per-tweet price, and any pipeline that counts rows will count fifteen tweets.</li>
<li><strong>apidojo Unlimited charged $0.0504 for one tweet.</strong> Its pricing has three event types: $0.016 per search or profile query (“includes first ~40 results”), $0.0004 per row above that, and $0.05 per single-tweet URL. The search and profile jobs cost the same as V2 because the query fee replaced the first 40 rows; the lookup did not. Both are in the run’s <span class="mono">chargedEventCounts</span>, and in the raw data below.</li>
<li><strong>Timings on a single lookup spread from 3 to 52 seconds.</strong> apidojo V2 answered the image post in 3 s and the long-form post in 52; xquik took 4 and 3; danek 4 and 6; scrape.badger 6 and 6; kaitoeasyapi 16 and 16; apidojo Unlimited 51 and 30. On the first run xquik answered both in 2 s and apidojo V2 took 16 and 4. One run per job, so treat a single lookup time as a sample, not a rate.</li>
<li>Metrics agree. All six report 312 likes and 53,562 views on the image post.</li>
</ul>
</section>"""


def sec_fields():
    rows = []
    for k, label in CANON_LABELS:
        cells = [label]
        for s in ORDER:
            f = run(s, "search")["items"]["fill"][k]
            if f is None:
                cells.append('<span class="faint">—</span>')
            elif f >= 99:
                cells.append('<span class="good-text">✓</span>')
            else:
                cells.append(f'<span class="warn-text">✓ when set</span>')
        rows.append(cells)
    heads = ["Field"] + [e(short(s).split(" / ")[0]) + "<br><span class='faint'>" + e(short(s).split(" / ")[1]) + "</span>" for s in ORDER]
    return f"""
<section class="post-section" id="fields">
<h2>7. Field coverage</h2>
<p>Read from the 100 search rows of each actor, after mapping every actor’s names onto one list. “✓ when set” means the key is present only when it has a value (an empty media list is omitted rather than written as <span class="mono">[]</span>). “—” means the actor never returns the field under any name.</p>
{table(heads, rows, cls="results-table fields")}
<ul class="plain">
<li><strong>Tweet URL</strong> is missing from danek and scrape.badger. Both give you the id and the handle, so you build it yourself.</li>
<li><strong>Retweet flag</strong> is missing from danek in this output; the nested <span class="mono">retweeted_tweet</span> object tells you anyway. xquik carries an explicit <span class="mono">isRetweet</span>.</li>
<li><strong>Long-form flag</strong> exists only in xquik (<span class="mono">isNoteTweet</span>).</li>
<li><strong>Source app</strong> is missing from kaitoeasyapi (the key exists, empty in every row).</li>
<li><strong>Naming.</strong> apidojo, kaitoeasyapi and xquik use camelCase; danek and scrape.badger use snake_case. danek returns three different record shapes for the three jobs (<span class="mono">user_info</span> on search, <span class="mono">author</span> on profile, <span class="mono">likes</span> instead of <span class="mono">favorites</span> on lookup) and returns <span class="mono">views</span> as a string. kaitoeasyapi’s record is apidojo’s record with four fields renamed, which makes the two nearly interchangeable.</li>
</ul>
</section>"""


def sec_filters():
    rows = []
    for s in ORDER:
        f = FILTERS[s]
        rows.append([actor_link(s), f'<span class="mono">{f["fields"]}</span>', f["required"], f["modes"], f["dates"], f["lang"], f["engagement"], f["media"], f["sort"], f["multi"]])
    notes = "".join(f"<li><strong>{e(short(s))}.</strong> {FILTERS[s]['notes']}</li>" for s in ORDER)
    return f"""
<section class="post-section" id="filters">
<h2>8. Filters and the shape of the input</h2>
<p>Read from each actor’s published input schema on 8 September 2026 and read again on 19 September: the field counts and the required fields did not change (the schemas as read on 19 September are in <a href="#raw">the raw data</a>). We did not exercise every filter; this is what the schema offers.</p>
{table(["Actor", "Input fields", "Required", "Targets / modes", "Date range", "Language", "Engagement filters", "Media filters", "Sort", "Several queries per run"], rows, cls="results-table wide")}
<ul class="plain">{notes}</ul>
</section>"""


def sec_cost():
    rows = []
    for s in ORDER:
        tot = sum((run(s, t) or {}).get("usd_total") or 0 for t in ("search", "profile", "lookup_media", "lookup_note"))
        listed = SEL[s]["listed"]
        charged = run(s, "search")["listed_price_per_item"]
        model = {"apidojo/tweet-scraper": "flat per row", "kaitoeasyapi/twitter-x-data-tweet-scraper-pay-per-result-cheapest": "flat per row, padded on empty or tiny results",
                 "apidojo/twitter-scraper-lite": "per query + per row tiers + per single-tweet URL", "xquik/x-tweet-scraper": "flat per row + platform usage billed to you",
                 "danek/twitter-scraper": "flat per row", "scrape.badger/twitter-tweets-scraper": "flat per row"}[s]
        rows.append([actor_link(s), f'<span class="mono">${listed:.5f}</span>', f'<span class="mono">${charged:.5f}</span>', model, f'<span class="mono">{usd(tot, 4)}</span>'])
    x = run("xquik/x-tweet-scraper", "search")
    plat = x["platform_usage_usd"]
    plat_pct = plat / (x["usd_total"] - plat) * 100
    px = PREV["runs"]["xquik/x-tweet-scraper|search"]
    prev_pct = px["platform_usage_usd"] / (px["usd_total"] - px["platform_usage_usd"]) * 100
    return f"""
<section class="post-section" id="cost">
<h2>9. Cost: what each Twitter scraper actually charged</h2>
{table(["Actor", "Listed per tweet", "Applied to our run", "Pricing model", "Our total, 4 runs"], rows)}
<ul class="plain">
<li><strong>The listed price is not always the applied price.</strong> Apify lets publishers set different per-event prices for the FREE, BRONZE, SILVER and GOLD account tiers; the store shows one number. Our Starter-plan account was charged $0.00022 by kaitoeasyapi (listed $0.00025), $0.00028 by danek (listed $0.0003) and $0.00014 by scrape.badger (listed $0.00015). The other three charged the listed price.</li>
<li><strong>xquik is the only actor here where platform usage is billed to the user.</strong> On the search run that was {usd(plat)} of compute, storage and transfer on top of $0.015 in per-tweet events: about {plat_pct:.0f}% ({prev_pct:.0f}% on the first run, when the same job took 34 seconds instead of 5). The store shows this as a small line under the price; the run object shows it as <span class="mono">platformUsageBillingModel: USER</span>. On a 256 MB run it is minor. On heavier settings it would not be.</li>
<li><strong>apidojo Unlimited’s per-query fee</strong> makes small pulls expensive and large pulls the same price as V2. Its single-tweet price is 125× the per-row price.</li>
<li><strong>kaitoeasyapi’s padding</strong> turns “no or few results” into a paid minimum, and the padding is billed as rows.</li>
</ul>
</section>"""


def sec_liked():
    cards = []
    for s in ORDER:
        good, bad = LIKED[s]
        cards.append(f"""<div class="card like-card"><h3>{actor_link(s, True)}</h3>
<div class="like-cols"><div><span class="label good-text">Liked</span><ul>{''.join(f'<li>{g}</li>' for g in good)}</ul></div>
<div><span class="label bad-text">Did not like</span><ul>{''.join(f'<li>{b}</li>' for b in bad)}</ul></div></div></div>""")
    return f"""
<section class="post-section" id="liked">
<h2>10. What we liked and what we did not</h2>
<div class="like-grid">{''.join(cards)}</div>
</section>"""


def sec_verdict():
    return f"""
<section class="post-section" id="verdict">
<h2>11. Best Twitter (X) scraper on Apify, by use case</h2>
<div class="verdict-card">
<span class="label">If you need</span>
<ul class="plain">
<li><strong>Keyword search at volume, full text, lowest cost:</strong> scrape.badger ($0.14 per 1,000, 18 s) or xquik ($0.16 all-in, 5 s). Both return long-form posts whole. Know that xquik’s 100 and the other five’s 100 differed on 17 ids in this run; if you need what X’s “Latest” tab shows, the five agree with each other and xquik did not.</li>
<li><strong>A profile timeline as X shows it, retweets included:</strong> xquik, apidojo V2 or danek. xquik returns it in date order with retweets and replies flagged and the original post attached; apidojo V2 and danek need a sort. danek is cheaper and attaches the original post too; apidojo V2 has the better filter set.</li>
<li><strong>Only an account’s original posts, no replies:</strong> no mode we ran does that now. Take a timeline actor and drop rows where the reply flag is set.</li>
<li><strong>Single-tweet lookups:</strong> xquik, danek or scrape.badger (3–7 s). Not apidojo Unlimited ($0.05 each) and not kaitoeasyapi (15 billed rows each).</li>
<li><strong>Long-form posts:</strong> any of the four that are not apidojo. Both apidojo actors cut at ~280 characters on every job we ran, on both dates.</li>
<li><strong>The most filters, cleanly named:</strong> apidojo V2.</li>
<li><strong>The smallest records:</strong> danek, at a quarter of xquik’s size, if you can live with three schemas.</li>
</ul>
</div>
<p>There is no single winner, and we did not expect one. The six were chosen to be different, and they are. The one result we did not expect on the first run still stands on the second: the oldest and most-used actor in the category, with 98,000 lifetime users, is the one that loses most of a long-form post, on a direct lookup as well as in search.</p>
<h3>Disclosure</h3>
<p>apifystats publishes no X actor and has no commercial relationship with any of the six publishers. Before the first run, none of them was contacted and none was told they were being compared. Since 9 September the plan, the inputs and the scripts have been public on this page, so any publisher can read them and change their actor with the test in mind; a re-run measures whatever is live on the day, and this one did. We paid list price from our own Apify account both times. This is the first post in a series; the method will change as we learn, and we will say so when it does.</p>
</section>"""


def sec_raw():
    schema_links = ", ".join(f'<a href="raw/schemas/{e(s.replace("/", "~"))}.input_schema.json">{e(short(s))}</a>' for s in ORDER)
    rows = []
    for s in ORDER:
        d = s.replace("/", "__")
        cells = [actor_link(s)]
        for t in ("search", "profile", "lookup_media", "lookup_note"):
            if run(s, t):
                cells.append(f'<a href="raw/{d}/{t}.input.json">input</a> · <a href="raw/{d}/{t}.run.json">run</a> · <a href="raw/{d}/{t}.items.json">rows</a>')
            else:
                cells.append("—")
        rows.append(cells)
    return f"""
<section class="post-section" id="raw">
<h2>12. Raw data</h2>
<p>Everything the actors returned, untouched, plus the run objects Apify keeps for each run (timing, memory, charged events, applied pricing) and the exact input we sent. One directory per actor. If you find a mistake in our reading of it, <a href="mailto:support@apifystats.com">tell us</a> and we will correct the page.</p>
{table(["Actor", "Keyword search", "Profile timeline", "Lookup: image post", "Lookup: long-form post"], rows)}
<ul class="plain">
<li><a href="raw/manifest.jsonl">manifest.jsonl</a> — one line per run: actor, job, run id, wall time, status.</li>
<li><a href="data.json">data.json</a> — every number on this page, as produced by the analysis script.</li>
<li><a href="raw/tools/plan.json">plan.json</a>, <a href="raw/tools/run_tests.py">run_tests.py</a>, <a href="raw/tools/mapping.py">mapping.py</a>, <a href="raw/tools/analyze.py">analyze.py</a>, <a href="raw/tools/build_post.py">build_post.py</a> — the plan with every input, the harness that ran the actors, the field mapping, the analysis, and the script that rendered this page (plus the census and DataForSEO helpers in the same folder).</li>
<li><a href="raw/census/candidates2.txt">census candidates</a>, <a href="raw/census/growth.txt">six-week growth</a>, <a href="raw/census/ranks.txt">store-search ranks</a>, <a href="raw/census/events.txt">title and price events</a> — the census extracts behind section 1, with <a href="raw/census/candidates2.sql">the SQL</a>.</li>
<li>Input schemas as read on {RUN_DATE}: {schema_links}.</li>
<li><a href="archive/{FIRST_RUN}/">archive/{FIRST_RUN}/</a> — the first run in full: the page as first published, its <span class="mono">data.json</span>, and every raw file. The text and the data are untouched; the archived page carries a banner, a noindex tag and repointed asset links so it renders from its folder.</li>
</ul>
<h3>Method notes and limits</h3>
<ul class="plain">
<li>One run per job per actor, at one moment, with one query and one handle, on each of two dates eleven days apart. Between the dates the timings moved on every actor, the text, price and schema findings held for five of the six, and one actor changed its search result set and the meaning of its profile mode (section 13). A timing on this page is one sample.</li>
<li>Timings include cold start and are measured by the platform, not by us. Memory was each actor’s default (128–512 MB).</li>
<li>Prices applied to our account reflect its plan tier. Yours may differ; the run object will tell you.</li>
<li>We did not test proxies, rate limits, very large pulls, or any filter beyond sort and cap.</li>
<li>Success rates and user counts in section 1 are the store’s public statistics as captured by our daily census; we did not audit them. Definitions are on the <a href="/methodology.html">methodology page</a>.</li>
</ul>
</section>"""


CSS = """
<style>
  .post { max-width: 1040px; margin: 0 auto; padding: 0 28px 60px; }
  .post-head { padding: 40px 0 26px; border-bottom: 1px solid var(--border-soft); }
  .post-head .kicker { font-family: "IBM Plex Mono", monospace; font-size: 12px; letter-spacing: 0.08em; text-transform: uppercase; color: var(--editorial-text); }
  .post-head h1 { margin: 10px 0 14px; font-size: 38px; line-height: 1.08; letter-spacing: -0.03em; font-weight: 600; max-width: 900px; }
  .post-head .lede { margin: 0; font-size: 17px; line-height: 1.6; color: var(--muted); max-width: 760px; }
  .post-head .byline { margin-top: 16px; font-family: "IBM Plex Mono", monospace; font-size: 12.5px; color: var(--muted-2); }
  .post .stat-strip { margin: 0 0 8px; border: 1px solid var(--border); border-radius: 10px; overflow: hidden; }
  .post .stat-tile { padding: 14px 16px; border-right: 1px solid var(--border); }
  .post .stat-tile .value { font-family: "IBM Plex Mono", monospace; font-size: 22px; font-weight: 600; }
  .post .stat-tile .sub { margin-top: 2px; }
  .time-note { margin: 18px 0 6px; padding: 14px 18px; border: 1px solid var(--border); border-left: 3px solid var(--editorial); border-radius: 8px; background: var(--editorial-wash); max-width: 840px; }
  .time-note .label { display: block; font-family: "IBM Plex Mono", monospace; font-size: 11px; letter-spacing: 0.08em; text-transform: uppercase; color: var(--editorial-text); margin-bottom: 6px; }
  .time-note p { margin: 0; font-size: 14.5px; line-height: 1.6; color: var(--ink); }
  .tldr { margin: 22px 0 6px; padding: 18px 22px; border: 1px solid var(--border); border-radius: 10px; background: var(--card); max-width: 840px; }
  .tldr .label { display: block; font-family: "IBM Plex Mono", monospace; font-size: 11px; letter-spacing: 0.08em; text-transform: uppercase; color: var(--accent-text); margin-bottom: 8px; }
  .tldr ul { margin: 0; padding-left: 20px; }
  .tldr li { font-size: 15px; line-height: 1.6; margin-bottom: 6px; }
  .tldr-note { margin: 10px 0 0; font-size: 13px; color: var(--muted-2); }
  details.faq { padding: 6px 14px; margin: 8px 0; max-width: 840px; }
  details.faq summary { list-style: none; }
  details.faq summary::-webkit-details-marker { display: none; }
  details.faq h3 { display: inline; margin: 0; font-size: 16px; }
  details.faq p { margin: 8px 0 6px; }
  .toc { display: flex; flex-wrap: wrap; gap: 6px 14px; padding: 18px 0 6px; font-size: 13px; }
  .toc a { color: var(--muted); }
  .post-section { padding: 30px 0 6px; border-top: 1px solid var(--border-soft); }
  .post-section h2 { font-size: 24px; letter-spacing: -0.02em; margin: 0 0 12px; font-weight: 600; }
  .post-section h3 { font-size: 16px; margin: 22px 0 8px; font-weight: 600; }
  .post-section p, .post-section li { font-size: 15px; line-height: 1.65; }
  .post-section p { margin: 0 0 12px; max-width: 800px; }
  ul.plain { padding-left: 20px; margin: 0 0 14px; max-width: 840px; }
  ul.plain li { margin: 0 0 8px; }
  ol.jobs { padding-left: 22px; margin: 0 0 14px; max-width: 800px; }
  ol.jobs li { margin: 0 0 8px; }
  .tbl { overflow-x: auto; margin: 6px 0 8px; border: 1px solid var(--border); border-radius: 8px; }
  .post .results-table td, .post .results-table th { padding: 8px 10px; vertical-align: top; font-size: 13px; line-height: 1.4; font-family: "IBM Plex Sans", system-ui, sans-serif; font-variant-numeric: tabular-nums; text-align: left; }
  .post .results-table td .mono, .post .results-table th.num, .post .results-table td.num { font-family: "IBM Plex Mono", monospace; }
  .post .results-table td .why { display: block; font-size: 12px; color: var(--muted-2); margin-top: 2px; line-height: 1.4; }
  .post .results-table th { text-align: left; white-space: nowrap; }
  .post .results-table td:first-child { min-width: 150px; }
  .slug { font-size: 11.5px; word-break: break-all; }
  .post .results-table.fields td, .post .results-table.fields th { text-align: center; }
  .post .results-table.fields td:first-child, .post .results-table.fields th:first-child { text-align: left; }
  .post .results-table.wide td { min-width: 120px; }
  .post .table-note { font-size: 12.5px; color: var(--muted-2); line-height: 1.5; margin: 0 0 14px; max-width: 840px; }
  .faint { color: var(--faint); }
  .good-text { color: var(--good-text); }
  .bad-text { color: var(--bad-text); }
  .warn-text { color: var(--warn-text); }
  .up { color: var(--good-text); } .down { color: var(--bad-text); }
  details { margin: 8px 0; border: 1px solid var(--border); border-radius: 8px; padding: 8px 14px; background: var(--bg-alt); }
  summary { cursor: pointer; font-size: 14px; font-weight: 500; }
  .inputs { display: grid; grid-template-columns: repeat(auto-fill, minmax(300px, 1fr)); gap: 10px; padding: 10px 0 4px; }
  .inp-name { font-size: 12.5px; font-weight: 600; margin-bottom: 4px; }
  .inp pre { margin: 0; padding: 10px; background: var(--card); border: 1px solid var(--border); border-radius: 6px; font-size: 11.5px; line-height: 1.4; overflow-x: auto; }
  q { font-style: italic; color: var(--muted); }
  .like-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(460px, 1fr)); gap: 14px; margin: 8px 0 14px; }
  .like-card { padding: 16px 18px; }
  .like-card h3 { margin: 0 0 10px; font-size: 15px; }
  .like-cols { display: grid; grid-template-columns: 1fr 1fr; gap: 14px; }
  .like-cols .label { display: block; font-size: 11px; letter-spacing: 0.08em; text-transform: uppercase; font-weight: 600; margin-bottom: 6px; }
  .like-cols ul { margin: 0; padding-left: 18px; }
  .like-cols li { font-size: 13.5px; line-height: 1.5; margin-bottom: 6px; }
  .post .verdict-card { padding: 18px 22px; margin: 10px 0 18px; }
  .post .verdict-card ul { margin-top: 8px; }
  @media (max-width: 720px) { .post { padding: 0 16px 40px; } .post-head h1 { font-size: 28px; } .like-grid { grid-template-columns: 1fr; } .like-cols { grid-template-columns: 1fr; } .post .stat-strip { grid-template-columns: repeat(2, 1fr); } }
</style>
"""


def main():
    tools = RAW / "tools"
    tools.mkdir(exist_ok=True)
    SKILL = ROOT / ".claude/skills/apifystats-comparison-post/scripts"
    for f in ("common.py", "run_tests.py", "refresh_runs.py", "analyze.py", "research.py", "census_candidates.py", "fetch_schemas.py", "check_links.py"):
        shutil.copy(SKILL / f, tools / f)
    for f in ("plan.json", "mapping.py", "build_post.py"):
        shutil.copy(HERE / f, tools / f)
    total = sum(v["usd_total"] or 0 for v in R.values())
    n_rows = sum(v["items"]["n"] for v in R.values())
    title = "Best Twitter (X) Scrapers on Apify, Tested in 2026"
    h1 = "Best Twitter (X) scrapers on Apify, tested: six tweet scrapers, the same three jobs, every byte kept"
    desc = "Six Apify tweet scrapers, same three jobs, same second: search, profile timeline, single tweet. Speed, cost as charged, full-text capture. Raw data included."
    url = "https://apifystats.com/blog/best-twitter-scraper-apify/"
    ld = {"@context": "https://schema.org", "@graph": [
        {"@type": "Article", "headline": title, "alternativeHeadline": h1, "description": desc, "datePublished": POST_DATE, "dateModified": UPDATED,
         "url": url, "mainEntityOfPage": url, "author": {"@type": "Organization", "name": "apifystats", "url": "https://apifystats.com/"},
         "publisher": {"@type": "Organization", "name": "apifystats", "url": "https://apifystats.com/"},
         "about": [{"@type": "SoftwareApplication", "name": short(s2), "url": store(s2)} for s2 in ORDER]},
        {"@type": "BreadcrumbList", "itemListElement": [
            {"@type": "ListItem", "position": 1, "name": "apifystats", "item": "https://apifystats.com/"},
            {"@type": "ListItem", "position": 2, "name": "Blog", "item": "https://apifystats.com/blog/"},
            {"@type": "ListItem", "position": 3, "name": title, "item": url}]},
        {"@type": "FAQPage", "mainEntity": [{"@type": "Question", "name": q, "acceptedAnswer": {"@type": "Answer", "text": strip_tags(a)}} for q, a in FAQ]}]}
    toc = [("selection", "How we chose"), ("method", "What we asked"), ("glance", "At a glance"), ("search", "Search"), ("profile", "Profile"), ("lookup", "Single tweet"),
           ("fields", "Fields"), ("filters", "Filters"), ("cost", "Cost"), ("liked", "Liked / not"), ("verdict", "Best by use case"), ("raw", "Raw data"), ("changes", "Changes"), ("faq", "FAQ")]
    body = f"""<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>{e(title)} — apifystats</title>
<meta property="article:published_time" content="{POST_DATE}">
<meta property="article:modified_time" content="{UPDATED}">
<meta name="description" content="{e(desc)}">
<link rel="canonical" href="{url}">
<meta property="og:title" content="{e(title)} — apifystats">
<meta property="og:description" content="{e(desc)}">
<meta property="og:type" content="article">
<meta property="og:site_name" content="apifystats">
<meta property="og:url" content="{url}">
<meta name="twitter:card" content="summary_large_image">
<link rel="icon" href="../../assets/favicon.svg?v={ASSET_V}" type="image/svg+xml">
<link rel="alternate icon" href="../../assets/favicon-32.png?v={ASSET_V}" type="image/png">
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=IBM+Plex+Sans:wght@400;500;600;700&family=IBM+Plex+Mono:wght@400;500;600&display=swap" rel="stylesheet">
<link rel="stylesheet" href="../../assets/design.css?v={ASSET_V}">
<script>
  try {{ var t = localStorage.getItem("theme"); if (t) document.documentElement.dataset.theme = t; }} catch (e) {{}}
</script>
{CSS}
<script type="application/ld+json">{json.dumps(ld)}</script>
</head>
<body class="blog-post">
<header class="site-header">
  <a class="wordmark" href="../../"><span class="datum"></span><span class="name">apifystats</span></a>
  <nav class="site-nav">
    <a href="../../search.html">Search</a>
    <a href="../../platforms/">Platforms</a>
    <a href="../../publishers/">Publishers</a>
    <a href="../../tasks/">Tasks</a>
    <a href="../../blog/" class="active">Blog</a>
  </nav>
  <div class="header-meta">
    <a class="ea-btn" href="/app/">Log in</a>
    <a class="cta-btn" href="/app/">Start free</a>
  </div>
</header>
<main class="post">
<article>
<div class="post-head">
  <div class="kicker">Deep comparison · No. 1 · X (Twitter) tweet scrapers · September 2026</div>
  <h1>{e(h1)}</h1>
  <p class="lede">There are hundreds of Twitter (X) scrapers on the Apify Store and no way to tell them apart from their listings. We picked six tweet scrapers, each for a different reason, gave them the same three jobs at the same second, and kept every byte they returned. This page explains how the six were chosen, what we sent, what came back, and where they differ. The raw data is at the bottom. Every run is dated, every run is kept, and the page is re-tested; the numbers here are from the run of {RUN_DATE}.</p>
  <div class="byline">apifystats · published {POST_DATE} · first run {FIRST_RUN} · re-tested {RUN_DATE} · census data of {DATA_DATE}</div>
</div>
<div class="toc">{''.join(f'<a href="#{a}">{n}</a>' for a, n in toc)}</div>
<div class="stat-strip">
  <div class="stat-tile"><div class="value">6</div><div class="sub">actors, from 5 publishers</div></div>
  <div class="stat-tile"><div class="value">24</div><div class="sub">runs on {RUN_DATE}, all SUCCEEDED</div></div>
  <div class="stat-tile"><div class="value">{n_rows:,}</div><div class="sub">rows returned</div></div>
  <div class="stat-tile"><div class="value">{usd(total, 2)}</div><div class="sub">charged in total</div></div>
</div>
{sec_tldr()}
{sec_time()}
{sec_selection()}
{sec_method()}
{sec_glance()}
{sec_search()}
{sec_profile()}
{sec_lookups()}
{sec_fields()}
{sec_filters()}
{sec_cost()}
{sec_liked()}
{sec_verdict()}
{sec_raw()}
{sec_changes()}
{sec_faq()}
</article>
</main>
<footer class="site-footer">
  <div class="brand">
    <a class="wordmark footer-wordmark" href="../../"><span class="datum"></span><span class="name">apifystats</span></a>
    <span>An independent statistics project. <strong>Not affiliated with, endorsed by, or sponsored by Apify.</strong> Prices and metrics are collected from public store listings and may lag the store.</span>
    <span class="mono" style="font-size:12px;">data: {DATA_DATE} &middot; support@apifystats.com &middot; <a href="/app/">Log in</a> &middot; <a href="/#pricing">Pricing</a> &middot; <a href="../../privacy.html">Privacy</a> &middot; <a href="../../terms.html">Terms</a></span>
    <div class="theme-toggle" id="theme-toggle">
      <button data-set="light">Light</button>
      <button data-set="dark">Dark</button>
      <button data-set="">OS</button>
    </div>
  </div>
  <div class="cols"><div class="col"><span class="label">Browse</span><a href="../../search.html">Search</a><a href="../../platforms/">Platforms</a><a href="../../tasks/">Tasks</a><a href="../../leaderboard.html">Leaderboard</a><a href="../../lists/">Lists</a><a href="../../publishers/">Publishers</a><a href="../../stats.html">Stats</a><a href="../../blog/">Blog</a><a href="../../methodology.html">Methodology</a></div><div class="col"><span class="label">Top platforms</span><a href="../../platforms/instagram-com.html">Instagram</a><a href="../../platforms/youtube-com.html">YouTube</a><a href="../../platforms/linkedin-com.html">LinkedIn</a><a href="../../platforms/google-com-maps.html">Google Maps</a><a href="../../platforms/tiktok-com.html">TikTok</a><a href="../../platforms/facebook-com.html">Facebook</a><a href="../../platforms/x-com.html">X (Twitter)</a><a href="../../platforms/amazon-com.html">Amazon</a></div></div>
</footer>
<script src="../../assets/theme.js?v={ASSET_V}" defer></script>
<script src="../../assets/ph.js?v={ASSET_V}"></script>
</body>
</html>
"""
    (OUTDIR / "index.html").write_text(body)
    print(f"wrote {OUTDIR/'index.html'} ({len(body):,} bytes); rows={n_rows} total=${total:.4f}")


if __name__ == "__main__":
    main()