Initial commit of Doormile CRM Mobile with UI modernizations
This commit is contained in:
28
qdrant/README.md
Normal file
28
qdrant/README.md
Normal file
@@ -0,0 +1,28 @@
|
||||
# Qdrant Production Setup Guide
|
||||
|
||||
## 1. How to run this on your Ubuntu VPS
|
||||
1. Copy this entire `qdrant` folder to your Ubuntu server.
|
||||
2. SSH into your server and navigate to this folder.
|
||||
3. Run the following command to start the database in the background:
|
||||
```bash
|
||||
docker compose up -d
|
||||
```
|
||||
|
||||
## 2. Accessing the UI Dashboard
|
||||
Qdrant comes with a built-in, beautiful dashboard.
|
||||
Once the container is running, open your web browser and go to:
|
||||
`http://<YOUR_VPS_IP_ADDRESS>:6333/dashboard`
|
||||
|
||||
Here you can:
|
||||
- See all your Collections.
|
||||
- View clusters and nodes.
|
||||
- Visually inspect your vector payloads and data.
|
||||
|
||||
## 3. Data Persistence
|
||||
We have configured two persistent volumes:
|
||||
- `./storage`: This is where the live database lives. Even if the server restarts, data is safe here.
|
||||
- `./snapshots`: Use this folder to store manual database backups.
|
||||
|
||||
## 4. Security Warning (Crucial)
|
||||
By default, port `6333` is open. You **must** configure your Ubuntu UFW firewall so that only your backend server (e.g., your Node.js or Python API server) can access this port.
|
||||
Do NOT leave it fully open to the public internet, or anyone can access your vectors.
|
||||
32
qdrant/docker-compose.yml
Normal file
32
qdrant/docker-compose.yml
Normal file
@@ -0,0 +1,32 @@
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
qdrant:
|
||||
image: qdrant/qdrant:latest
|
||||
container_name: qdrant_production
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
# REST API & Web Dashboard (Accessible at http://<VPS_IP>:6333/dashboard)
|
||||
- "6333:6333"
|
||||
# gRPC API (For faster, low-latency communication with backends)
|
||||
- "6334:6334"
|
||||
volumes:
|
||||
# Persistent Volume: Ensures data is never lost on restarts
|
||||
- ./storage:/qdrant/storage
|
||||
# Snapshots Volume: Dedicated folder for backups
|
||||
- ./snapshots:/qdrant/snapshots
|
||||
environment:
|
||||
# Disable telemetry for privacy
|
||||
- QDRANT__TELEMETRY_DISABLED=true
|
||||
- QDRANT__SERVICE__API_KEY=Package@321#
|
||||
ulimits:
|
||||
# Professional tuning: Vector DBs need high open file limits for disk I/O
|
||||
nofile:
|
||||
soft: 65535
|
||||
hard: 65535
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
# Memory Limit: Prevents Qdrant from taking 100% of your VPS RAM.
|
||||
# Adjust '4G' based on your actual VPS specs.
|
||||
memory: 4G
|
||||
84
qdrant/migrate.py
Normal file
84
qdrant/migrate.py
Normal file
@@ -0,0 +1,84 @@
|
||||
import os
|
||||
from qdrant_client import QdrantClient
|
||||
from qdrant_client.models import PointStruct
|
||||
|
||||
# ==========================================
|
||||
# 1. SETUP YOUR CONNECTIONS
|
||||
# ==========================================
|
||||
# Connect to your old Qdrant Cloud (Source)
|
||||
cloud_client = QdrantClient(
|
||||
url="https://369d1a18-9eaa-4bf9-be05-115c9f188387.us-west-1-0.aws.cloud.qdrant.io:6333",
|
||||
api_key="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3MiOiJtIiwic3ViamVjdCI6ImFwaS1rZXk6MDY3MmFiZDktOTRhZi00NWMyLWJhZDctNWZlNmI2MTRiZDBkIn0.z_En6ORE7oHyGRxBAAjkho8-b-cqTXbWOPUassDfbYw"
|
||||
)
|
||||
|
||||
# Connect to your new VPS Server (Destination)
|
||||
vps_client = QdrantClient(url="http://66.116.207.225:6333")
|
||||
|
||||
# We will loop through both of your collections!
|
||||
COLLECTIONS_TO_MIGRATE = ["doormile_clients", "doormile_auth"]
|
||||
|
||||
for collection_name in COLLECTIONS_TO_MIGRATE:
|
||||
print(f"\n==========================================")
|
||||
print(f"🚀 Starting migration for: {collection_name}")
|
||||
print(f"==========================================")
|
||||
|
||||
# ==========================================
|
||||
# 2. RECREATE COLLECTION ON VPS
|
||||
# ==========================================
|
||||
try:
|
||||
collection_info = cloud_client.get_collection(collection_name=collection_name)
|
||||
except Exception as e:
|
||||
print(f"Could not find collection '{collection_name}' on cloud. Skipping...")
|
||||
continue
|
||||
|
||||
print(f"Recreating '{collection_name}' on your VPS with identical settings...")
|
||||
vps_client.recreate_collection(
|
||||
collection_name=collection_name,
|
||||
vectors_config=collection_info.config.params.vectors,
|
||||
)
|
||||
|
||||
# ==========================================
|
||||
# 3. TRANSFER THE DATA IN BATCHES
|
||||
# ==========================================
|
||||
batch_size = 100
|
||||
offset = None
|
||||
total_migrated = 0
|
||||
|
||||
print("Starting data transfer...")
|
||||
|
||||
while True:
|
||||
# Fetch a batch of points from the cloud
|
||||
records, offset = cloud_client.scroll(
|
||||
collection_name=collection_name,
|
||||
limit=batch_size,
|
||||
offset=offset,
|
||||
with_payload=True,
|
||||
with_vectors=True
|
||||
)
|
||||
|
||||
if not records:
|
||||
break
|
||||
|
||||
# Convert records to PointStruct format
|
||||
points = [
|
||||
PointStruct(
|
||||
id=record.id,
|
||||
vector=record.vector,
|
||||
payload=record.payload
|
||||
)
|
||||
for record in records
|
||||
]
|
||||
|
||||
# Upload the batch to your VPS
|
||||
vps_client.upsert(
|
||||
collection_name=collection_name,
|
||||
points=points
|
||||
)
|
||||
|
||||
total_migrated += len(points)
|
||||
print(f" -> Copied {total_migrated} vectors...")
|
||||
|
||||
if offset is None:
|
||||
break
|
||||
|
||||
print("\n✅ Migration Complete! Both collections are safely on your VPS.")
|
||||
126
qdrant/qdrant_to_postgres.py
Normal file
126
qdrant/qdrant_to_postgres.py
Normal file
@@ -0,0 +1,126 @@
|
||||
import psycopg2
|
||||
from qdrant_client import QdrantClient
|
||||
from psycopg2.extras import execute_values
|
||||
import json
|
||||
|
||||
# ==========================================
|
||||
# 1. SETUP YOUR CONNECTIONS
|
||||
# ==========================================
|
||||
|
||||
# Connect to your Qdrant (using your Cloud connection details with API Key)
|
||||
print("🔌 Connecting to Qdrant...")
|
||||
qdrant_client = QdrantClient(
|
||||
url="https://369d1a18-9eaa-4bf9-be05-115c9f188387.us-west-1-0.aws.cloud.qdrant.io:6333",
|
||||
api_key="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3MiOiJtIiwic3ViamVjdCI6ImFwaS1rZXk6MDY3MmFiZDktOTRhZi00NWMyLWJhZDctNWZlNmI2MTRiZDBkIn0.z_En6ORE7oHyGRxBAAjkho8-b-cqTXbWOPUassDfbYw"
|
||||
)
|
||||
|
||||
# Connect to your PostgreSQL Database (Credentials from your .env)
|
||||
print("🔌 Connecting to PostgreSQL...")
|
||||
try:
|
||||
pg_conn = psycopg2.connect(
|
||||
host="31.97.228.132",
|
||||
port="5433",
|
||||
dbname="logistics",
|
||||
user="admin",
|
||||
password="Package@321#"
|
||||
)
|
||||
pg_cursor = pg_conn.cursor()
|
||||
print("✅ PostgreSQL Connected!")
|
||||
except Exception as e:
|
||||
print(f"❌ Failed to connect to Postgres: {e}")
|
||||
exit(1)
|
||||
|
||||
# ==========================================
|
||||
# 2. MIGRATION FUNCTIONS
|
||||
# ==========================================
|
||||
|
||||
def migrate_clients():
|
||||
print("\n🚀 Starting migration for: doormile_clients")
|
||||
collection_name = "doormile_clients"
|
||||
|
||||
# Scroll through all points in Qdrant
|
||||
offset = None
|
||||
batch_size = 100
|
||||
total_migrated = 0
|
||||
|
||||
while True:
|
||||
records, offset = qdrant_client.scroll(
|
||||
collection_name=collection_name,
|
||||
limit=batch_size,
|
||||
offset=offset,
|
||||
with_payload=True,
|
||||
with_vectors=False # We don't need vectors for Postgres!
|
||||
)
|
||||
|
||||
if not records:
|
||||
break
|
||||
|
||||
for record in records:
|
||||
payload = record.payload
|
||||
|
||||
# Extract fields safely from payload (Fallback to empty string if missing)
|
||||
first_name = payload.get("first_name", payload.get("name", "Unknown"))
|
||||
last_name = payload.get("last_name", "")
|
||||
phone = payload.get("phone", "")
|
||||
address = payload.get("address", "")
|
||||
|
||||
# We skip survey_lat/long here because old data likely doesn't have it,
|
||||
# but we can add default 0.0 values if required by the database schema.
|
||||
survey_lat = payload.get("survey_lat", 0.0)
|
||||
survey_long = payload.get("survey_long", 0.0)
|
||||
pincode = payload.get("pincode", "")
|
||||
registration_source = payload.get("registration_source", "legacy_qdrant")
|
||||
|
||||
# Insert into PostgreSQL
|
||||
try:
|
||||
pg_cursor.execute("""
|
||||
INSERT INTO doormile_clients
|
||||
(first_name, last_name, phone, address, surveylat, surveylong, pincode, registration_source, created_at, updated_at)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, NOW(), NOW())
|
||||
RETURNING id;
|
||||
""", (first_name, last_name, phone, address, survey_lat, survey_long, pincode, registration_source))
|
||||
|
||||
# Fetch the newly generated Postgres ID
|
||||
new_client_id = pg_cursor.fetchone()[0]
|
||||
|
||||
# Now we check if there's an email/password in this same payload,
|
||||
# if so, we migrate it to the auth table simultaneously!
|
||||
email = payload.get("email")
|
||||
password = payload.get("password") # Note: You should hash this in Go, but here we preserve legacy passwords
|
||||
|
||||
if email and password:
|
||||
pg_cursor.execute("""
|
||||
INSERT INTO doormile_auth
|
||||
(client_id, email, password_hash, role, created_at, updated_at)
|
||||
VALUES (%s, %s, %s, 'user', NOW(), NOW())
|
||||
ON CONFLICT (email) DO NOTHING;
|
||||
""", (new_client_id, email, password))
|
||||
|
||||
pg_conn.commit()
|
||||
total_migrated += 1
|
||||
|
||||
except Exception as e:
|
||||
pg_conn.rollback()
|
||||
print(f"⚠️ Error inserting record {record.id}: {e}")
|
||||
|
||||
print(f" -> Migrated {total_migrated} clients to Postgres...")
|
||||
|
||||
if offset is None:
|
||||
break
|
||||
|
||||
# ==========================================
|
||||
# 3. RUN THE MIGRATION
|
||||
# ==========================================
|
||||
|
||||
print("\n==========================================")
|
||||
print("📦 MIGRATION INITIALIZED")
|
||||
print("==========================================")
|
||||
|
||||
# Execute the migration
|
||||
migrate_clients()
|
||||
|
||||
# Close connections
|
||||
pg_cursor.close()
|
||||
pg_conn.close()
|
||||
|
||||
print("\n✅ Migration Complete! All legacy data is now strictly relational in PostgreSQL.")
|
||||
Reference in New Issue
Block a user