|
# retoor <retoor@molodetz.nl>
|
|
|
|
from devplacepy.services.jobs.deepsearch.extract import extract_html, relevant_links
|
|
|
|
PAGE = """
|
|
<html><head><title>Framework Trends - Blog</title></head><body>
|
|
<header><a href="/">Home</a> <a href="/about">About</a> <a href="/login">Login</a></header>
|
|
<nav><ul><li><a href="/blog">Blog</a></li><li><a href="/dev">Dev</a></li></ul></nav>
|
|
<div role="banner">We use cookies to improve your experience on this website today.</div>
|
|
<main><article>
|
|
<h1>Trends that define web development</h1>
|
|
<p>Server components became the default rendering model, cutting client bundles by forty percent according to the survey.</p>
|
|
<p>Signals-based reactivity landed in the standards pipeline; see the <a href="/signals-deep-dive">signals deep dive</a> article.</p>
|
|
</article></main>
|
|
<footer><p>Copyright. All rights reserved. Privacy. Terms. Subscribe to our newsletter now.</p></footer>
|
|
</body></html>
|
|
"""
|
|
|
|
|
|
def test_extract_prefers_article_content_and_drops_chrome():
|
|
page = extract_html(PAGE, base_url="https://blog.example.com/trends/")
|
|
assert page.title == "Framework Trends - Blog"
|
|
assert "Server components" in page.text
|
|
assert "Signals-based reactivity" in page.text
|
|
assert "cookies" not in page.text
|
|
assert "Copyright" not in page.text
|
|
assert "Home" not in page.text
|
|
|
|
|
|
def test_extract_emits_blank_line_paragraphs():
|
|
page = extract_html(PAGE, base_url="https://blog.example.com/trends/")
|
|
assert "\n\n" in page.text
|
|
|
|
|
|
def test_extract_resolves_links_absolute_and_skips_nav():
|
|
page = extract_html(PAGE, base_url="https://blog.example.com/trends/")
|
|
urls = [url for url, _text in page.links]
|
|
assert "https://blog.example.com/signals-deep-dive" in urls
|
|
assert "https://blog.example.com/blog" not in urls
|
|
|
|
|
|
def test_extract_unescapes_entities():
|
|
page = extract_html(
|
|
"<html><body><p>Ampersand & arrow → and more text to pass the length gate.</p></body></html>"
|
|
)
|
|
assert "&" not in page.text
|
|
assert "→" not in page.text
|
|
assert "&" in page.text
|
|
|
|
|
|
def test_extract_survives_malformed_html():
|
|
page = extract_html("<div><p>Unclosed paragraph with enough characters to be kept around.<div></span>")
|
|
assert "Unclosed paragraph" in page.text
|
|
|
|
|
|
def test_relevant_links_scores_by_query_overlap():
|
|
links = [
|
|
("https://a.example/web-frameworks-2026", "web frameworks in 2026"),
|
|
("https://a.example/cookie-policy", "cookie policy"),
|
|
("https://a.example/logo.png", "frameworks logo"),
|
|
]
|
|
picked = relevant_links(links, "latest web frameworks 2026", 2)
|
|
assert picked == ["https://a.example/web-frameworks-2026"]
|
|
|
|
|
|
def test_relevant_links_empty_query_returns_nothing():
|
|
assert relevant_links([("https://a.example/x", "text")], "", 3) == []
|