GET api update for image vector
This commit is contained in:
@@ -43,9 +43,12 @@ Read-only: nothing here writes, and the pipeline stages are untouched.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import logging
|
||||
import math
|
||||
import re
|
||||
import struct
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional, Sequence, Set, Tuple
|
||||
|
||||
@@ -117,6 +120,66 @@ def normalise_vector(vector: Sequence[float]) -> List[float]:
|
||||
return [v / norm for v in values]
|
||||
|
||||
|
||||
def encode_vector_b64(vector: Sequence[float]) -> str:
|
||||
"""The compact query-string form: urlsafe base64 of EMBED_DIM little-endian
|
||||
float32 values, no padding. 5,464 characters for 1024 floats, against
|
||||
8-10 KB for comma-separated decimals - the difference between fitting
|
||||
every proxy's request-line limit and not. Client-side equivalents are in
|
||||
docs/IMAGE_SEARCH_API.md."""
|
||||
values = [float(v) for v in vector]
|
||||
if len(values) != EMBED_DIM:
|
||||
raise InvalidVectorError(f"vector must have {EMBED_DIM} values, got {len(values)}")
|
||||
packed = struct.pack(f"<{EMBED_DIM}f", *values)
|
||||
return base64.urlsafe_b64encode(packed).decode("ascii").rstrip("=")
|
||||
|
||||
|
||||
def parse_vector_param(raw: str) -> List[float]:
|
||||
"""The `vector` query parameter of GET /search/image-vector, as floats.
|
||||
|
||||
Two encodings, told apart by the presence of a comma:
|
||||
|
||||
* comma-separated decimals - what a log line or a print() gives you;
|
||||
surrounding brackets, whitespace and newlines are ignored;
|
||||
* base64 (urlsafe or standard alphabet, padding optional) of exactly
|
||||
EMBED_DIM little-endian float32 - what `encode_vector_b64` produces.
|
||||
|
||||
Only the shape is checked here; length, NaN and zero-norm are
|
||||
`normalise_vector`'s job, so every route reports them the same way.
|
||||
"""
|
||||
text = (raw or "").strip()
|
||||
if text[:1] == "[" and text[-1:] == "]":
|
||||
text = text[1:-1]
|
||||
if not text.strip():
|
||||
raise InvalidVectorError("vector is empty")
|
||||
|
||||
if "," in text:
|
||||
values: List[float] = []
|
||||
for position, piece in enumerate(text.split(","), start=1):
|
||||
piece = piece.strip()
|
||||
if not piece:
|
||||
continue
|
||||
try:
|
||||
values.append(float(piece))
|
||||
except ValueError:
|
||||
raise InvalidVectorError(f"value {position} ({piece[:20]!r}) is not a number") from None
|
||||
return values
|
||||
|
||||
compact = "".join(text.split())
|
||||
padded = compact + "=" * (-len(compact) % 4)
|
||||
try:
|
||||
packed = base64.urlsafe_b64decode(padded.replace("+", "-").replace("/", "_"))
|
||||
except (ValueError, binascii.Error):
|
||||
raise InvalidVectorError(
|
||||
f"vector must be {EMBED_DIM} comma-separated numbers or base64 of {EMBED_DIM} float32 values"
|
||||
) from None
|
||||
if len(packed) != EMBED_DIM * 4:
|
||||
raise InvalidVectorError(
|
||||
f"base64 vector decodes to {len(packed)} bytes, expected {EMBED_DIM * 4} "
|
||||
f"({EMBED_DIM} little-endian float32)"
|
||||
)
|
||||
return list(struct.unpack(f"<{EMBED_DIM}f", packed))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# label text
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user