Paginate comments without duplicates

Request the first page

Start without a cursor. The complete note URL and its current token remain unchanged for the entire chain.

import os
import requests

base_url = "https://xhs-data-api.p.rapidapi.com"
headers = {
    "X-RapidAPI-Key": os.environ["X_RAPIDAPI_KEY"],
    "X-RapidAPI-Host": "xhs-data-api.p.rapidapi.com",
}
note_url = os.environ["XHS_NOTE_URL"]

response = requests.get(
    f"{base_url}/note/comments_from_url",
    headers=headers,
    params={"url": note_url, "count": 10, "raw": "false"},
    timeout=45,
)
response.raise_for_status()
page = response.json()["data"]

page["count"] equals len(page["items"]). Continue only when has_more is true and next_cursor is present.

Continue and deduplicate

cursor = None
seen = set()

while True:
    params = {"url": note_url, "count": 10, "raw": "false"}
    if cursor:
        params["cursor"] = cursor

    response = requests.get(
        f"{base_url}/note/comments_from_url",
        headers=headers,
        params=params,
        timeout=45,
    )
    response.raise_for_status()
    page = response.json()["data"]

    for comment in page["items"]:
        if comment["comment_id"] not in seen:
            seen.add(comment["comment_id"])
            print(comment["comment_id"], comment.get("content"))

    if not page["has_more"]:
        break
    cursor = page["next_cursor"]

Normalized like_count and sub_comment_count are integers when supplied upstream. Comment status is an upstream integer code, not one fixed enum.