247 lines
13 KiB
Python
247 lines
13 KiB
Python
"""Recommendations under a product's ratings and reviews (docs/RECOMMENDATIONS.md,
|
|
Phase 1). Offline scoring tests first; the database tests are skipped when the
|
|
local Postgres container is not running."""
|
|
from __future__ import annotations
|
|
|
|
from decimal import Decimal
|
|
|
|
import pytest
|
|
|
|
from app.electronics import recommend as rec
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Scoring
|
|
# ---------------------------------------------------------------------------
|
|
def test_bayesian_rating_trusts_many_ratings_over_few():
|
|
mean = 4.0
|
|
few_perfect = rec.bayesian_rating(5.0, 3, mean)
|
|
many_good = rec.bayesian_rating(4.4, 2000, mean)
|
|
assert many_good > few_perfect
|
|
assert rec.bayesian_rating(None, None, mean) == mean # unrated: the pool average
|
|
assert rec.bayesian_rating(5.0, None, mean) == rec.bayesian_rating(5.0, 1, mean)
|
|
|
|
|
|
def test_price_closeness():
|
|
assert rec.price_closeness(20000, 20000) == 1.0
|
|
assert rec.price_closeness(15000, 20000) == pytest.approx(0.75)
|
|
assert rec.price_closeness(45000, 20000) == 0.0 # never negative
|
|
assert rec.price_closeness(20000, None) == 0.5 # unknown target price
|
|
|
|
|
|
def test_similarity_dominates_but_rating_and_price_count():
|
|
target = {"best_price": 20000.0, "rating": 4.1}
|
|
close = {"product_id": 1, "similarity": 0.95, "best_price": 21000.0, "rating": 4.0, "rating_count": 500}
|
|
far = {"product_id": 2, "similarity": 0.60, "best_price": 20000.0, "rating": 4.8, "rating_count": 5000}
|
|
tie_better_rated = {"product_id": 3, "similarity": 0.95, "best_price": 21000.0, "rating": 4.6, "rating_count": 3000}
|
|
out = rec.recommend(target, [far, close, tie_better_rated], [])
|
|
assert [c["product_id"] for c in out] == [3, 1, 2]
|
|
assert out[0]["reason"] == "Similar specs · 4.6★ vs 4.1★"
|
|
assert out[1]["reason"] == "Similar specs · 4.0★ (500 ratings)"
|
|
assert all(c["basis"] == "similar" for c in out)
|
|
assert rec.reason({"rating": 4.0, "rating_count": 1}, None, "similar") == "Similar specs · 4.0★ (1 rating)"
|
|
|
|
|
|
def test_limit_and_top_rated_fill_when_few_similar():
|
|
target = {"best_price": 20000.0, "rating": None}
|
|
similar = [{"product_id": 1, "similarity": 0.9, "best_price": 20000.0, "rating": None, "rating_count": None}]
|
|
rated = [
|
|
{"product_id": 1, "best_price": 20000.0, "rating": 4.9, "rating_count": 9000}, # already picked
|
|
{"product_id": 2, "best_price": 30000.0, "rating": 4.2, "rating_count": 900},
|
|
{"product_id": 3, "best_price": 25000.0, "rating": 4.7, "rating_count": 1200},
|
|
{"product_id": 4, "best_price": 25000.0, "rating": None, "rating_count": None}, # unrated: never a "top rated"
|
|
]
|
|
out = rec.recommend(target, similar, rated, limit=3)
|
|
assert [c["product_id"] for c in out] == [1, 3, 2]
|
|
assert [c["basis"] for c in out] == ["similar", "top_rated", "top_rated"]
|
|
assert out[0]["reason"] == "Similar specs"
|
|
assert out[1]["reason"] == "Top rated in this category · 4.7★ (1,200 ratings)"
|
|
|
|
|
|
def test_enough_similar_means_no_fill():
|
|
similar = [{"product_id": i, "similarity": 0.5, "best_price": 1.0, "rating": None, "rating_count": None}
|
|
for i in range(rec.MIN_MATCHES)]
|
|
rated = [{"product_id": 99, "best_price": 1.0, "rating": 5.0, "rating_count": 10}]
|
|
out = rec.recommend({"best_price": 1.0, "rating": None}, similar, rated)
|
|
assert 99 not in {c["product_id"] for c in out}
|
|
|
|
|
|
def test_better_rated_rules():
|
|
target = {"rating": 4.1}
|
|
rated = [
|
|
{"product_id": 1, "rating": 4.5, "rating_count": 4000},
|
|
{"product_id": 2, "rating": 4.9, "rating_count": 4}, # too few ratings
|
|
{"product_id": 3, "rating": 4.1, "rating_count": 9000}, # not higher
|
|
{"product_id": 4, "rating": 4.6, "rating_count": 3000},
|
|
{"product_id": 5, "rating": None, "rating_count": None},
|
|
]
|
|
out = rec.better_rated(target, rated)
|
|
assert [c["product_id"] for c in out] == [4, 1]
|
|
assert out[0]["reason"] == "4.6★ vs 4.1★ · 3,000 ratings"
|
|
# Unrated product: any well-rated product counts as better rated.
|
|
out = rec.better_rated({"rating": None}, rated)
|
|
assert [c["product_id"] for c in out] == [4, 1, 3]
|
|
assert out[0]["reason"] == "4.6★ · 3,000 ratings"
|
|
|
|
|
|
def test_better_rated_tie_goes_to_more_positive_reviews():
|
|
rated = [
|
|
{"product_id": 1, "rating": 4.5, "rating_count": 1000, "sentiment": {"positive": 2, "negative": 8}},
|
|
{"product_id": 2, "rating": 4.5, "rating_count": 1000, "sentiment": {"positive": 8, "negative": 2}},
|
|
{"product_id": 3, "rating": 4.5, "rating_count": 1000}, # no stored reviews: neutral
|
|
]
|
|
assert [c["product_id"] for c in rec.better_rated({"rating": 4.0}, rated)] == [2, 3, 1]
|
|
assert rec.sentiment_balance({"positive": 3, "neutral": 1}) == 0.75
|
|
assert rec.sentiment_balance({}) == 0.0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# API (database)
|
|
# ---------------------------------------------------------------------------
|
|
def _vector(*head: float) -> list:
|
|
v = list(head) + [0.0] * (384 - len(head))
|
|
norm = sum(x * x for x in v) ** 0.5
|
|
return [x / norm for x in v]
|
|
|
|
|
|
def _seed():
|
|
"""Samsung phones on two sites each, plus embeddings:
|
|
S24 8/256 (target, 4.1★, ₹74,999), S24 8/128 (its variant), S23 (closest,
|
|
4.5★ from 4,000), S22 (further, 4.3★ from only 4), A55 (close in specs but
|
|
₹39,999 - outside both price bands), and an out-of-stock Z Flip6."""
|
|
from app.electronics.collector import Collector, RunOptions, RunStats
|
|
from app.electronics.db import repository as repo
|
|
from app.electronics.db.connection import connect
|
|
from app.electronics.models import Listing
|
|
from app.electronics.normalise.title_parser import parse_title, variant_key
|
|
|
|
c = Collector.__new__(Collector)
|
|
c.opt = RunOptions(category="mobiles", brands=["samsung"])
|
|
c.ids, c.run_id, c._touched_products, c.stats = repo.id_maps(), None, {}, RunStats()
|
|
|
|
def store(title, sku, price, *, rating=None, count=None):
|
|
p = parse_title(title, "mobiles")
|
|
for site in ("amazon.in", "croma.com"):
|
|
l = Listing(site_domain=site, source_sku=f"{site}-{sku}", source_url=f"https://www.{site}/p/{sku}",
|
|
source_type="search_snippet", brand_slug="samsung", category="mobiles", title=title,
|
|
evidence_text=f"{title} ₹{price}", confidence=0.5, parser="test",
|
|
model=p.model, ram_gb=p.ram_gb, storage_gb=p.storage_gb, price=Decimal(price))
|
|
l.model_norm, l.variant_key = p.model_norm, variant_key(p, "mobiles")
|
|
l.rating, l.review_count = rating, count
|
|
c.store(l)
|
|
|
|
store("Samsung Galaxy S24 5G (8GB RAM, 256GB)", "s24-256", 74999, rating=Decimal("4.1"), count=300)
|
|
store("Samsung Galaxy S24 5G (8GB RAM, 128GB)", "s24-128", 69999)
|
|
store("Samsung Galaxy S23 5G (8GB RAM, 256GB)", "s23", 64999, rating=Decimal("4.5"), count=2000)
|
|
store("Samsung Galaxy S22 5G (8GB RAM, 256GB)", "s22", 79999, rating=Decimal("4.3"), count=2)
|
|
store("Samsung Galaxy A55 5G (8GB RAM, 128GB)", "a55", 39999, rating=Decimal("4.2"), count=800)
|
|
store("Samsung Galaxy Z Flip6 5G (12GB RAM, 256GB)", "flip6", 59999)
|
|
repo.refresh_verification()
|
|
|
|
with connect() as conn: # every phone has its own price
|
|
by_price = {int(r["best_price"]): r["product_id"] for r in conn.execute(
|
|
"SELECT product_id, best_price FROM elec.v_brand_catalog")}
|
|
pids = {"s24": by_price[74999], "s24_128": by_price[69999], "s23": by_price[64999],
|
|
"s22": by_price[79999], "a55": by_price[39999], "flip6": by_price[59999]}
|
|
with connect(autocommit=True) as conn: # the Flip6 sells out after it was verified
|
|
conn.execute("UPDATE elec.source_listing SET in_stock = FALSE WHERE source_sku LIKE '%%-flip6'")
|
|
vectors = {"s24": _vector(1, 0), "s24_128": _vector(1, 0), "s23": _vector(1, 0.2),
|
|
"s22": _vector(1, 1), "a55": _vector(1, 0.1), "flip6": _vector(1, 0.1)}
|
|
for key, pid in pids.items():
|
|
repo.set_embedding(pid, vectors[key])
|
|
return pids
|
|
|
|
|
|
def test_api_recommends_similar_in_stock_products_without_variants(db, client):
|
|
pids = _seed()
|
|
body = client.get(f"/api/elec/products/{pids['s24']}/recommendations").json()
|
|
got = [i["product_id"] for i in body["items"]]
|
|
# Closest first; the variant, the out-of-stock Flip6 and the A55 (outside
|
|
# the price limit, though close in specs) are left out.
|
|
assert got == [pids["s23"], pids["s22"]]
|
|
s23 = body["items"][0]
|
|
assert s23["rating"] == 4.5 and s23["rating_count"] == 4000 # 2,000 on each of two sites
|
|
assert s23["reason"] == "Similar specs · 4.5★ vs 4.1★"
|
|
assert s23["best_price"] == "64999.00"
|
|
assert [v["product_id"] for v in body["other_variants"]] == [pids["s24_128"]]
|
|
|
|
|
|
def test_api_recommendations_unknown_product_and_bad_type(db, client):
|
|
assert client.get("/api/elec/products/999999/recommendations").status_code == 404
|
|
pids = _seed()
|
|
assert client.get(f"/api/elec/products/{pids['s24']}/recommendations",
|
|
params={"type": "cheapest"}).status_code == 422
|
|
|
|
|
|
def test_api_falls_back_to_top_rated_without_embeddings(db, client):
|
|
from app.electronics.db.connection import connect
|
|
|
|
pids = _seed()
|
|
with connect(autocommit=True) as conn:
|
|
conn.execute("UPDATE elec.product SET embedding = NULL")
|
|
items = client.get(f"/api/elec/products/{pids['s24']}/recommendations").json()["items"]
|
|
assert [i["product_id"] for i in items] == [pids["s23"], pids["s22"]]
|
|
assert all(i["basis"] == "top_rated" for i in items)
|
|
|
|
|
|
def test_api_better_rated_needs_higher_rating_enough_reviews_and_close_price(db, client):
|
|
pids = _seed()
|
|
body = client.get(f"/api/elec/products/{pids['s24']}/recommendations", params={"type": "better_rated"}).json()
|
|
# S22 is rated higher but by only 4 people; A55 is outside +/-20% of the price.
|
|
assert body["type"] == "better_rated"
|
|
assert [i["product_id"] for i in body["items"]] == [pids["s23"]]
|
|
assert body["items"][0]["reason"] == "4.5★ vs 4.1★ · 4,000 ratings"
|
|
assert body["items"][0]["basis"] == "better_rated"
|
|
|
|
|
|
def test_product_list_carries_the_overall_rating_for_card_badges(db, client):
|
|
pids = _seed()
|
|
products = {p["product_id"]: p for p in
|
|
client.get("/api/elec/products", params={"category": "mobiles"}).json()["products"]}
|
|
assert (products[pids["s23"]]["rating"], products[pids["s23"]]["rating_count"]) == (4.5, 4000)
|
|
assert (products[pids["s24_128"]]["rating"], products[pids["s24_128"]]["rating_count"]) == (None, None)
|
|
|
|
|
|
def test_laptop_variants_need_the_same_processor(db, client):
|
|
"""One laptop line ("HP 15") spans many CPUs: only another RAM/storage of the
|
|
same processor is a variant. Other CPUs, and part-number-only listings that
|
|
state no processor, are separate products that can be recommended."""
|
|
from app.electronics.collector import Collector, RunOptions, RunStats
|
|
from app.electronics.db import repository as repo
|
|
from app.electronics.db.connection import connect
|
|
from app.electronics.models import Listing
|
|
from app.electronics.normalise.title_parser import parse_title, variant_key
|
|
|
|
c = Collector.__new__(Collector)
|
|
c.opt = RunOptions(category="laptops", brands=["hp"])
|
|
c.ids, c.run_id, c._touched_products, c.stats = repo.id_maps(), None, {}, RunStats()
|
|
for title, price in (("HP 15 Laptop AMD Ryzen 3 7320U (8GB RAM, 512GB SSD)", 40000),
|
|
("HP 15 Laptop AMD Ryzen 3 7320U (16GB RAM, 512GB SSD)", 45000),
|
|
("HP 15 Laptop AMD Ryzen 5 7520U (8GB RAM, 512GB SSD)", 47000),
|
|
("HP 15 Laptop 15-FC0805AU (8GB RAM, 512GB SSD)", 41000),
|
|
("HP 15 Laptop 15-FD0682TU (16GB RAM, 512GB SSD)", 42000)):
|
|
p = parse_title(title, "laptops")
|
|
for site in ("amazon.in", "croma.com"):
|
|
l = Listing(site_domain=site, source_sku=f"{site}-{price}", source_url=f"https://www.{site}/p/{price}",
|
|
source_type="search_snippet", brand_slug="hp", category="laptops", title=title,
|
|
evidence_text=f"{title} ₹{price}", confidence=0.5, parser="test", model=p.model,
|
|
model_number=p.mpn, ram_gb=p.ram_gb, storage_gb=p.storage_gb, price=Decimal(price))
|
|
l.model_norm, l.variant_key, l.processor = p.model_norm, variant_key(p, "laptops"), p.processor
|
|
c.store(l)
|
|
repo.refresh_verification()
|
|
with connect() as conn:
|
|
by_price = {int(r["best_price"]): r["product_id"] for r in conn.execute(
|
|
"SELECT product_id, best_price FROM elec.v_brand_catalog")}
|
|
r3_8, r3_16, r5, fc, fd = (by_price[n] for n in (40000, 45000, 47000, 41000, 42000))
|
|
for i, pid in enumerate((r3_8, r3_16, r5, fc, fd)):
|
|
repo.set_embedding(pid, _vector(1, 0.1 * i))
|
|
|
|
body = client.get(f"/api/elec/products/{r3_8}/recommendations").json()
|
|
assert [v["product_id"] for v in body["other_variants"]] == [r3_16]
|
|
assert {r5, fc, fd} <= {i["product_id"] for i in body["items"]}
|
|
assert r3_16 not in {i["product_id"] for i in body["items"]}
|
|
|
|
body = client.get(f"/api/elec/products/{fc}/recommendations").json()
|
|
assert body["other_variants"] == []
|
|
assert fd in {i["product_id"] for i in body["items"]}
|