Add WebSocket, multi API key, geocoding proxy, beacon map picker, and comprehensive bug fixes

- Multi API Key + permission system (read/write/admin) with SHA-256 hash
- WebSocket real-time push (location, alarm, device_status, attendance, bluetooth)
- Geocoding proxy endpoints for Amap POI search and reverse geocode
- Beacon modal map-based location picker with search and click-to-select
- GCJ-02 ↔ WGS-84 bidirectional coordinate conversion
- Data cleanup scheduler (configurable retention days)
- Fix GPS longitude sign inversion (course_status bit 11: 0=East, 1=West)
- Fix 2G CellID 2→3 bytes across all protocols (0x28, 0x2C, parser.py)
- Fix parser loop guards, alarm_source field length, CommandLog.sent_at
- Fix geocoding IMEI parameterization, require_admin import
- Improve API error messages for 422 validation errors
- Remove beacon floor/area fields (consolidated into name)

via [HAPI](https://hapi.run)

Co-Authored-By: HAPI <noreply@hapi.run>
This commit is contained in:
2026-03-24 05:10:05 +00:00
parent 7d6040af41
commit 11281e5be2
24 changed files with 1636 additions and 730 deletions

View File

@@ -2,34 +2,77 @@
Geocoding service - Convert cell tower / WiFi AP data to lat/lon coordinates,
and reverse geocode coordinates to addresses.
Uses free APIs:
- Cell tower: Google Geolocation API (if key available) or unwiredlabs.com
- WiFi: Same APIs support WiFi AP lookup
- Reverse geocoding: 天地图 (Tianditu) - free, WGS84 native
All services use 高德 (Amap) API exclusively.
- Forward geocoding (cell/WiFi → coords): 高德智能硬件定位
- Reverse geocoding (coords → address): 高德逆地理编码
"""
import json
import hashlib
import logging
import math
from collections import OrderedDict
from typing import Optional
from urllib.parse import quote
import aiohttp
logger = logging.getLogger(__name__)
# Import keys from centralized config (no more hardcoded values here)
from app.config import settings as _settings
GOOGLE_API_KEY: Optional[str] = _settings.GOOGLE_API_KEY
UNWIRED_API_TOKEN: Optional[str] = _settings.UNWIRED_API_TOKEN
TIANDITU_API_KEY: Optional[str] = _settings.TIANDITU_API_KEY
BAIDU_MAP_AK: Optional[str] = _settings.BAIDU_MAP_AK
AMAP_KEY: Optional[str] = _settings.AMAP_KEY
AMAP_SECRET: Optional[str] = _settings.AMAP_SECRET
# Maximum cache entries (LRU eviction) — configurable via settings
_CACHE_MAX_SIZE = _settings.GEOCODING_CACHE_SIZE
# ---------------------------------------------------------------------------
# WGS-84 → GCJ-02 coordinate conversion (server-side)
# ---------------------------------------------------------------------------
_A = 6378245.0
_EE = 0.00669342162296594
def _out_of_china(lat: float, lon: float) -> bool:
return not (73.66 < lon < 135.05 and 3.86 < lat < 53.55)
def _transform_lat(x: float, y: float) -> float:
ret = -100.0 + 2.0 * x + 3.0 * y + 0.2 * y * y + 0.1 * x * y + 0.2 * math.sqrt(abs(x))
ret += (20.0 * math.sin(6.0 * x * math.pi) + 20.0 * math.sin(2.0 * x * math.pi)) * 2.0 / 3.0
ret += (20.0 * math.sin(y * math.pi) + 40.0 * math.sin(y / 3.0 * math.pi)) * 2.0 / 3.0
ret += (160.0 * math.sin(y / 12.0 * math.pi) + 320.0 * math.sin(y * math.pi / 30.0)) * 2.0 / 3.0
return ret
def _transform_lon(x: float, y: float) -> float:
ret = 300.0 + x + 2.0 * y + 0.1 * x * x + 0.1 * x * y + 0.1 * math.sqrt(abs(x))
ret += (20.0 * math.sin(6.0 * x * math.pi) + 20.0 * math.sin(2.0 * x * math.pi)) * 2.0 / 3.0
ret += (20.0 * math.sin(x * math.pi) + 40.0 * math.sin(x / 3.0 * math.pi)) * 2.0 / 3.0
ret += (150.0 * math.sin(x / 12.0 * math.pi) + 300.0 * math.sin(x / 30.0 * math.pi)) * 2.0 / 3.0
return ret
def wgs84_to_gcj02(lat: float, lon: float) -> tuple[float, float]:
"""Convert WGS-84 to GCJ-02 (used by 高德)."""
if _out_of_china(lat, lon):
return (lat, lon)
d_lat = _transform_lat(lon - 105.0, lat - 35.0)
d_lon = _transform_lon(lon - 105.0, lat - 35.0)
rad_lat = lat / 180.0 * math.pi
magic = math.sin(rad_lat)
magic = 1 - _EE * magic * magic
sqrt_magic = math.sqrt(magic)
d_lat = (d_lat * 180.0) / ((_A * (1 - _EE)) / (magic * sqrt_magic) * math.pi)
d_lon = (d_lon * 180.0) / (_A / sqrt_magic * math.cos(rad_lat) * math.pi)
return (lat + d_lat, lon + d_lon)
# ---------------------------------------------------------------------------
# LRU Cache
# ---------------------------------------------------------------------------
class LRUCache(OrderedDict):
"""Simple LRU cache based on OrderedDict."""
@@ -51,13 +94,29 @@ class LRUCache(OrderedDict):
self.popitem(last=False)
# Cache cell tower lookups to avoid redundant API calls
_cell_cache: LRUCache = LRUCache()
_wifi_cache: LRUCache = LRUCache()
# Cache reverse geocoding results (coord rounded to ~100m -> address)
_address_cache: LRUCache = LRUCache()
# ---------------------------------------------------------------------------
# 高德数字签名 (AMAP_SECRET)
# ---------------------------------------------------------------------------
def _amap_sign(params: dict) -> str:
"""Generate 高德 API digital signature (MD5)."""
if not AMAP_SECRET:
return ""
sorted_str = "&".join(f"{k}={params[k]}" for k in sorted(params.keys()))
raw = sorted_str + AMAP_SECRET
return hashlib.md5(raw.encode()).hexdigest()
# ===========================================================================
# Forward Geocoding: cell/WiFi → lat/lon
# ===========================================================================
async def geocode_location(
mcc: Optional[int] = None,
mnc: Optional[int] = None,
@@ -65,272 +124,129 @@ async def geocode_location(
cell_id: Optional[int] = None,
wifi_list: Optional[list[dict]] = None,
neighbor_cells: Optional[list[dict]] = None,
imei: Optional[str] = None,
) -> tuple[Optional[float], Optional[float]]:
"""
Convert cell tower and/or WiFi AP data to lat/lon.
Parameters
----------
mcc : int - Mobile Country Code
mnc : int - Mobile Network Code
lac : int - Location Area Code
cell_id : int - Cell Tower ID
wifi_list : list[dict] - WiFi APs [{"mac": "AA:BB:CC:DD:EE:FF", "signal": -70}, ...]
neighbor_cells : list[dict] - Neighbor cells [{"lac": ..., "cell_id": ..., "rssi": ...}, ...]
Returns
-------
(latitude, longitude) or (None, None)
Uses 高德智能硬件定位 API exclusively.
"""
# Check cache first (cell tower)
# Check cache first
if mcc is not None and lac is not None and cell_id is not None:
cache_key = (mcc, mnc or 0, lac, cell_id)
cached = _cell_cache.get_cached(cache_key)
if cached is not None:
return cached
# Try Google Geolocation API first
if GOOGLE_API_KEY:
result = await _geocode_google(mcc, mnc, lac, cell_id, wifi_list, neighbor_cells)
if AMAP_KEY:
result = await _geocode_amap(mcc, mnc, lac, cell_id, wifi_list, neighbor_cells, imei=imei)
if result[0] is not None:
if mcc is not None and lac is not None and cell_id is not None:
_cell_cache.put((mcc, mnc or 0, lac, cell_id), result)
return result
# Try Unwired Labs API
if UNWIRED_API_TOKEN:
result = await _geocode_unwired(mcc, mnc, lac, cell_id, wifi_list, neighbor_cells)
if result[0] is not None:
if mcc is not None and lac is not None and cell_id is not None:
_cell_cache.put((mcc, mnc or 0, lac, cell_id), result)
return result
# Fallback: Mylnikov.org (free, no API key required)
if mcc is not None and lac is not None and cell_id is not None:
result = await _geocode_mylnikov_cell(mcc, mnc or 0, lac, cell_id)
if result[0] is not None:
_cell_cache.put((mcc, mnc or 0, lac, cell_id), result)
return result
# Try WiFi via Mylnikov
if wifi_list:
for ap in wifi_list:
mac = ap.get("mac", "")
if mac:
result = await _geocode_mylnikov_wifi(mac)
if result[0] is not None:
return result
return (None, None)
async def _geocode_google(
mcc, mnc, lac, cell_id, wifi_list, neighbor_cells
async def _geocode_amap(
mcc, mnc, lac, cell_id, wifi_list, neighbor_cells, *, imei: Optional[str] = None
) -> tuple[Optional[float], Optional[float]]:
"""Use Google Geolocation API."""
url = f"https://www.googleapis.com/geolocation/v1/geolocate?key={GOOGLE_API_KEY}"
body: dict = {}
"""
Use 高德智能硬件定位 API (apilocate.amap.com/position).
if mcc is not None:
body["homeMobileCountryCode"] = mcc
if mnc is not None:
body["homeMobileNetworkCode"] = mnc
Returns coordinates (高德 returns GCJ-02).
"""
# Build bts (base station) parameter: mcc,mnc,lac,cellid,signal
bts = ""
if mcc is not None and lac is not None and cell_id is not None:
bts = f"{mcc},{mnc or 0},{lac},{cell_id},-65"
# Cell towers
towers = []
if lac is not None and cell_id is not None:
towers.append({
"cellId": cell_id,
"locationAreaCode": lac,
"mobileCountryCode": mcc or 0,
"mobileNetworkCode": mnc or 0,
})
# Build nearbts (neighbor cells)
nearbts_parts = []
if neighbor_cells:
for nc in neighbor_cells:
towers.append({
"cellId": nc.get("cell_id", 0),
"locationAreaCode": nc.get("lac", 0),
"mobileCountryCode": mcc or 0,
"mobileNetworkCode": mnc or 0,
"signalStrength": -(nc.get("rssi", 0)),
})
if towers:
body["cellTowers"] = towers
nc_lac = nc.get("lac", 0)
nc_cid = nc.get("cell_id", 0)
nc_signal = -(nc.get("rssi", 0)) if nc.get("rssi") else -80
nearbts_parts.append(f"{mcc or 460},{mnc or 0},{nc_lac},{nc_cid},{nc_signal}")
# WiFi APs
# Build macs (WiFi APs): mac,signal,ssid
macs_parts = []
if wifi_list:
aps = []
for ap in wifi_list:
aps.append({
"macAddress": ap.get("mac", ""),
"signalStrength": -(ap.get("signal", 0)),
})
body["wifiAccessPoints"] = aps
mac = ap.get("mac", "").lower().replace(":", "")
signal = -(ap.get("signal", 0)) if ap.get("signal") else -70
ssid = ap.get("ssid", "")
macs_parts.append(f"{mac},{signal},{ssid}")
if not bts and not macs_parts:
return (None, None)
params = {"accesstype": "0", "imei": imei or _settings.GEOCODING_DEFAULT_IMEI, "key": AMAP_KEY}
if bts:
params["bts"] = bts
if nearbts_parts:
params["nearbts"] = "|".join(nearbts_parts)
if macs_parts:
params["macs"] = "|".join(macs_parts)
# Add digital signature
sig = _amap_sign(params)
if sig:
params["sig"] = sig
url = "https://apilocate.amap.com/position"
try:
async with aiohttp.ClientSession() as session:
async with session.post(url, json=body, timeout=aiohttp.ClientTimeout(total=5)) as resp:
if resp.status == 200:
data = await resp.json()
loc = data.get("location", {})
lat = loc.get("lat")
lng = loc.get("lng")
if lat is not None and lng is not None:
logger.info("Google geocode: lat=%.6f, lon=%.6f", lat, lng)
return (lat, lng)
else:
text = await resp.text()
logger.warning("Google geocode failed: %d %s", resp.status, text[:200])
except Exception as e:
logger.warning("Google geocode error: %s", e)
return (None, None)
async def _geocode_unwired(
mcc, mnc, lac, cell_id, wifi_list, neighbor_cells
) -> tuple[Optional[float], Optional[float]]:
"""Use Unwired Labs LocationAPI."""
url = "https://us1.unwiredlabs.com/v2/process.php"
body: dict = {"token": UNWIRED_API_TOKEN}
# Cell towers
cells = []
if mcc is not None and lac is not None and cell_id is not None:
cells.append({
"lac": lac,
"cid": cell_id,
"mcc": mcc,
"mnc": mnc or 0,
})
if neighbor_cells:
for nc in neighbor_cells:
cells.append({
"lac": nc.get("lac", 0),
"cid": nc.get("cell_id", 0),
"mcc": mcc or 0,
"mnc": mnc or 0,
"signal": -(nc.get("rssi", 0)),
})
if cells:
body["cells"] = cells
# WiFi APs
if wifi_list:
aps = []
for ap in wifi_list:
aps.append({
"bssid": ap.get("mac", ""),
"signal": -(ap.get("signal", 0)),
})
body["wifi"] = aps
try:
async with aiohttp.ClientSession() as session:
async with session.post(url, json=body, timeout=aiohttp.ClientTimeout(total=5)) as resp:
if resp.status == 200:
data = await resp.json()
if data.get("status") == "ok":
lat = data.get("lat")
lon = data.get("lon")
if lat is not None and lon is not None:
logger.info("Unwired geocode: lat=%.6f, lon=%.6f", lat, lon)
return (lat, lon)
else:
logger.warning("Unwired geocode: %s", data.get("message", "unknown error"))
except Exception as e:
logger.warning("Unwired geocode error: %s", e)
return (None, None)
async def _geocode_mylnikov_cell(
mcc: int, mnc: int, lac: int, cell_id: int
) -> tuple[Optional[float], Optional[float]]:
"""Use Mylnikov.org free cell tower geocoding API (no API key required)."""
url = (
f"https://api.mylnikov.org/geolocation/cell"
f"?v=1.1&data=open"
f"&mcc={mcc}&mnc={mnc}&lac={lac}&cellid={cell_id}"
)
try:
async with aiohttp.ClientSession() as session:
async with session.get(url, timeout=aiohttp.ClientTimeout(total=5)) as resp:
async with session.get(
url, params=params, timeout=aiohttp.ClientTimeout(total=5)
) as resp:
if resp.status == 200:
data = await resp.json(content_type=None)
if data.get("result") == 200:
lat = data.get("data", {}).get("lat")
lon = data.get("data", {}).get("lon")
if lat is not None and lon is not None:
logger.info("Mylnikov cell geocode: lat=%.6f, lon=%.6f", lat, lon)
if data.get("status") == "1" and data.get("result"):
result = data["result"]
location = result.get("location", "")
if location and "," in location:
lon_str, lat_str = location.split(",")
lat = float(lat_str)
lon = float(lon_str)
logger.info("Amap geocode: lat=%.6f, lon=%.6f", lat, lon)
return (lat, lon)
else:
logger.debug("Mylnikov cell: no result for MCC=%d MNC=%d LAC=%d CellID=%d",
mcc, mnc, lac, cell_id)
infocode = data.get("infocode", "")
if infocode == "10012":
logger.debug("Amap geocode: insufficient permissions (enterprise cert needed)")
else:
logger.warning("Amap geocode error: %s (code=%s)", data.get("info", ""), infocode)
else:
logger.warning("Mylnikov cell API HTTP %d", resp.status)
logger.warning("Amap geocode HTTP %d", resp.status)
except Exception as e:
logger.warning("Mylnikov cell geocode error: %s", e)
logger.warning("Amap geocode error: %s", e)
return (None, None)
async def _geocode_mylnikov_wifi(mac: str) -> tuple[Optional[float], Optional[float]]:
"""Use Mylnikov.org free WiFi AP geocoding API."""
# Normalize MAC format (needs colons)
mac = mac.upper().replace("-", ":")
url = f"https://api.mylnikov.org/geolocation/wifi?v=1.1&data=open&bssid={mac}"
try:
async with aiohttp.ClientSession() as session:
async with session.get(url, timeout=aiohttp.ClientTimeout(total=5)) as resp:
if resp.status == 200:
data = await resp.json(content_type=None)
if data.get("result") == 200:
lat = data.get("data", {}).get("lat")
lon = data.get("data", {}).get("lon")
if lat is not None and lon is not None:
logger.info("Mylnikov WiFi geocode: lat=%.6f, lon=%.6f (MAC=%s)", lat, lon, mac)
_wifi_cache.put(mac, (lat, lon))
return (lat, lon)
else:
logger.debug("Mylnikov WiFi API HTTP %d for MAC=%s", resp.status, mac)
except Exception as e:
logger.warning("Mylnikov WiFi geocode error: %s", e)
return (None, None)
# ---------------------------------------------------------------------------
# Reverse Geocoding: coordinates -> address
# ---------------------------------------------------------------------------
# ===========================================================================
# Reverse Geocoding: coordinates → address
# ===========================================================================
async def reverse_geocode(
lat: float, lon: float
) -> Optional[str]:
"""
Convert lat/lon to a human-readable address.
Convert lat/lon (WGS-84) to a human-readable address.
Priority: Baidu Map > Tianditu (fallback).
Both accept WGS84 coordinates natively (Baidu via coordtype=wgs84ll).
Returns None if no reverse geocoding service is available.
Uses 高德逆地理编码 API exclusively.
"""
# Round to ~100m for cache key to reduce API calls
cache_key = (round(lat, 3), round(lon, 3))
cached = _address_cache.get_cached(cache_key)
if cached is not None:
return cached
# Try Baidu Map first (higher quality addresses for China)
if BAIDU_MAP_AK:
result = await _reverse_geocode_baidu(lat, lon)
if result:
_address_cache.put(cache_key, result)
return result
# Fallback to Tianditu
if TIANDITU_API_KEY:
result = await _reverse_geocode_tianditu(lat, lon)
if AMAP_KEY:
result = await _reverse_geocode_amap(lat, lon)
if result:
_address_cache.put(cache_key, result)
return result
@@ -338,101 +254,55 @@ async def reverse_geocode(
return None
async def _reverse_geocode_baidu(
async def _reverse_geocode_amap(
lat: float, lon: float
) -> Optional[str]:
"""
Use Baidu Map reverse geocoding API.
Use 高德逆地理编码 API.
API docs: https://lbsyun.baidu.com/faq/api?title=webapi/guide/webservice-geocoding
Input coordtype: wgs84ll (WGS-84, same as GPS data, no conversion needed).
Free tier: 5,000 requests/day (personal developer).
API: https://restapi.amap.com/v3/geocode/regeo
Input: GCJ-02 coordinates (need to convert from WGS-84).
Free tier: 5,000 requests/day (personal), 1,000,000/day (enterprise).
"""
url = (
f"https://api.map.baidu.com/reverse_geocoding/v3/"
f"?ak={BAIDU_MAP_AK}&output=json&coordtype=wgs84ll"
f"&location={lat},{lon}"
)
gcj_lat, gcj_lon = wgs84_to_gcj02(lat, lon)
params = {
"key": AMAP_KEY,
"location": f"{gcj_lon:.6f},{gcj_lat:.6f}",
"extensions": "base",
"output": "json",
}
sig = _amap_sign(params)
if sig:
params["sig"] = sig
url = "https://restapi.amap.com/v3/geocode/regeo"
try:
async with aiohttp.ClientSession() as session:
async with session.get(
url, timeout=aiohttp.ClientTimeout(total=5)
url, params=params, timeout=aiohttp.ClientTimeout(total=5)
) as resp:
if resp.status == 200:
data = await resp.json(content_type=None)
if data.get("status") == 0:
result = data.get("result", {})
formatted = result.get("formatted_address", "")
if formatted:
# Add sematic_description for more context
sematic = result.get("sematic_description", "")
address = formatted
if sematic and sematic not in formatted:
address = f"{formatted} ({sematic})"
if data.get("status") == "1":
regeocode = data.get("regeocode", {})
formatted = regeocode.get("formatted_address", "")
if formatted and formatted != "[]":
logger.info(
"Baidu reverse geocode: %.6f,%.6f -> %s",
lat, lon, address,
"Amap reverse geocode: %.6f,%.6f -> %s",
lat, lon, formatted,
)
return address
return formatted
else:
logger.warning(
"Baidu reverse geocode error: status=%s, msg=%s",
data.get("status"), data.get("message", ""),
"Amap reverse geocode error: info=%s, infocode=%s",
data.get("info", ""), data.get("infocode", ""),
)
else:
logger.warning("Baidu reverse geocode HTTP %d", resp.status)
logger.warning("Amap reverse geocode HTTP %d", resp.status)
except Exception as e:
logger.warning("Baidu reverse geocode error: %s", e)
return None
async def _reverse_geocode_tianditu(
lat: float, lon: float
) -> Optional[str]:
"""
Use 天地图 (Tianditu) reverse geocoding API.
API docs: http://lbs.tianditu.gov.cn/server/geocoding.html
Coordinate system: WGS84 (same as our GPS data, no conversion needed).
Free tier: 10,000 requests/day.
"""
post_str = json.dumps({"lon": lon, "lat": lat, "ver": 1}, separators=(",", ":"))
url = (
f"http://api.tianditu.gov.cn/geocoder"
f"?postStr={quote(post_str)}&type=geocode&tk={TIANDITU_API_KEY}"
)
try:
async with aiohttp.ClientSession() as session:
async with session.get(
url, timeout=aiohttp.ClientTimeout(total=5)
) as resp:
if resp.status == 200:
data = await resp.json(content_type=None)
if data.get("status") == "0":
result = data.get("result", {})
# Build address from components
addr_comp = result.get("addressComponent", {})
formatted = result.get("formatted_address", "")
if formatted:
# Add nearby POI if available
poi = addr_comp.get("poi", "")
address = formatted
if poi and poi not in formatted:
address = f"{formatted} ({poi})"
logger.info(
"Tianditu reverse geocode: %.6f,%.6f -> %s",
lat, lon, address,
)
return address
else:
logger.warning(
"Tianditu reverse geocode error: %s",
data.get("msg", data),
)
else:
logger.warning("Tianditu reverse geocode HTTP %d", resp.status)
except Exception as e:
logger.warning("Tianditu reverse geocode error: %s", e)
logger.warning("Amap reverse geocode error: %s", e)
return None