Private RAG Architecture: Deploying Retrieval Systems Inside Your VPC Without Data Leakage

flowchart TD
    subgraph PublicInternet ["External Network / Corporate VPN"]
        ClientApp["Authenticated Enterprise Client<br/>(Internal Web / ERP / CRM)"]
    end
    subgraph AWS_VPC ["Air-Gapped Cloud VPC (10.100.0.0/16)"]
        subgraph DMZ_Subnet ["DMZ Subnet (10.100.1.0/24)"]
            WAF["Cloud WAF & DDoS Shield"]
            ALB["Internal ALB / Reverse Proxy<br/>(mTLS + JWT Validation)"]
        end
        subgraph Compute_Subnet ["Private Isolated Compute Subnet (10.100.10.0/24) - ZERO EGRESS"]
            IngestService["Document Ingestion Engine<br/>(Docling / Layout Parser Pods)"]
            TEI["Local Embedding Service<br/>(BAAI/bge-m3 on NVIDIA L4)"]
            Reranker["Local Cross-Encoder<br/>(bge-reranker-large)"]
            vLLM["Air-Gapped Local LLM Engine<br/>(Llama-3.1-70B-AWQ on 2x A100 80GB)"]
            RAGOrchestrator["RAG Query & Synthesis Gateway<br/>(Context Assembly & Guardrails)"]
        end
        subgraph Data_Subnet ["Private Isolated Data Subnet (10.100.20.0/24)"]
            PGVector["PostgreSQL 16 Cluster + pgvector<br/>(HNSW Vector Index + BM25 tsvector)"]
            S3_VPCE["S3 VPC Gateway Endpoint<br/>(Raw Encrypted PDFs & Document Store)"]
            KMS_VPCE["AWS KMS Interface Endpoint<br/>(Customer-Managed Keys - CMK)"]
        end
    end
    subgraph Blocked_Perimeter ["Perimeter Enforcement"]
        IGW["Internet Gateway / NAT Egress<br/>[ROUTE 0.0.0.0/0: DROP / DENY]"]
    end
    ClientApp -->|Mutual TLS 1.3 + OIDC| WAF
    WAF --> ALB
    ALB --> RAGOrchestrator
    RAGOrchestrator -->|Internal gRPC| TEI
    RAGOrchestrator -->|SQL over TLS + RLS| PGVector
    RAGOrchestrator -->|Rank Pool| Reranker
    RAGOrchestrator -->|Local HTTP Streaming| vLLM
    IngestService -->|Read Raw Docs| S3_VPCE
    IngestService -->|Batch Embeddings| TEI
    IngestService -->|ACID Multi-Row Upsert| PGVector
    PGVector -.->|EBS Encryption| KMS_VPCE
    Compute_Subnet -.->|BLOCKED| IGW

+---------------------------------------------------------------------------------------------------+
|                            ENTERPRISE AIR-GAPPED VPC PERIMETER (10.100.0.0/16)                    |
+---------------------------------------------------------------------------------------------------+
|                                                                                                   |
|  [Corporate Network / VPN]                                                                        |
|            │ (Mutual TLS 1.3 + RBAC Token)                                                        |
|            ▼                                                                                      |
|  +───────────────────────────────────────────+                                                    |
|  | DMZ Subnet: Internal WAF + Envoy Gateway  |                                                    |
|  +───────────────────────────────────────────+                                                    |
|            │                                                                                      |
|            ▼ (Private VPC Network Interface)                                                      |
|  +─────────────────────────────────────────────────────────────────────────────────────────────+  |
|  | PRIVATE COMPUTE SUBNET (Zero Public Egress Route Table: 0.0.0.0/0 -> DENY)                  |  |
|  |                                                                                             |  |
|  |   ┌───────────────────────────┐      ┌───────────────────────────┐                          |  |
|  |   │ Document Layout Parser    │ ───► │ Local Embedding Engine    │                          |  |
|  |   │ (Docling Table Extraction)│      │ (BGE-M3 / TEI on GPU)     │                          |  |
|  |   └─────────────┬─────────────┘      └─────────────┬─────────────┘                          |  |
|  |                 │                                  │ (1024-dim vectors)                     |  |
|  |                 ▼                                  ▼                                        |  |
|  |   ┌──────────────────────────────────────────────────────────────┐                          |  |
|  |   │ RAG Orchestrator (Guardrails, RRF Fusion, Citation Tracking) │                          |  |
|  |   └─────────────────────────────┬────────────────────────────────┘                          |  |
|  |                                 │                                                           |  |
|  |                                 ▼                                                           |  |
|  |                  ┌──────────────────────────────┐                                           |  |
|  |                  │ Air-Gapped vLLM Engine       │                                           |  |
|  |                  │ (Llama-3.1-70B-AWQ / 4-bit)  │                                           |  |
|  |                  └──────────────────────────────┘                                           |  |
|  +─────────────────────────────────────────────────────────────────────────────────────────────+  |
|            │                                                                                      |
|            ▼ (PrivateLink / VPC Peering)                                                          |
|  +─────────────────────────────────────────────────────────────────────────────────────────────+  |
|  | PRIVATE DATA SUBNET                                                                         |  |
|  |   ┌─────────────────────────────────────────┐   ┌────────────────────────────────────────┐  |  |
|  |   │ PostgreSQL 16 + pgvector                │   │ Encrypted S3 Document Vault            │  |  |
|  |   │ (HNSW Cosine Index + BM25 Full-Text)    │   │ (Via S3 VPC Gateway Endpoint)          │  |  |
|  |   └─────────────────────────────────────────┘   └────────────────────────────────────────┘  |  |
|  +─────────────────────────────────────────────────────────────────────────────────────────────+  |
|                                                                                                   |
|  [FIREWALL EGRESS AUDIT]: Outbound Internet Gateway (IGW) = NON-EXISTENT                          |
|                           Public NAT Gateway = DISABLED                                           |
|                           DNS Resolver = Private Route 53 (External Domains Blackholed)           |
+---------------------------------------------------------------------------------------------------+

NAIVE CHUNKING FAILURE:
Chunk 1: "...Total Operating Expenses for Fiscal Year 2025 were detailed as follows: Salaries and Wages"
[CHUNK BOUNDARY BREAK]
Chunk 2: "$14,820,000. Rent and Utilities: $2,140,000. Depreciation: $840,000..."
--> LLM Query: "What were the salaries for 2025?" 
--> Result: Retrieval mismatch or inverted column mapping.

import io
import json
import uuid
import psycopg2
from psycopg2.extras import execute_values
import requests
from docling.document_converter import DocumentConverter
  # Configuration pointing to VPC internal endpoints
PG_DSN = "postgresql://rag_app:StrongVpcPassword@10.100.20.15:5432/enterprise_rag?sslmode=verify-full"
TEI_EMBED_URL = "http://tei-bge-m3.compute.internal:8080/embed"
converter = DocumentConverter()
def extract_structured_chunks(pdf_bytes: bytes, doc_id: str, tenant_id: str):
    """
    Parses complex multi-column PDFs into layout-preserved semantic chunks.
    Preserves tables as pristine markdown structures with spatial coordinates.
    """
    result = converter.convert(io.BytesIO(pdf_bytes))
    doc = result.document
    
    chunks = []
    current_section = "Introduction"
    
    for item in doc.iterate_items():
        # Track hierarchical headings
        if item.label in ["section_header", "heading"]:
            current_section = item.text.strip()
            continue
            
        # Deterministic table preservation
        if item.label == "table":
            table_md = item.export_to_markdown()
            chunk_text = f"Context: {current_section}\n\n[TABLE DATA]:\n{table_md}"
            bbox = item.prov[0].bbox if item.prov else None
            page_no = item.prov[0].page_no if item.prov else 1
            
            chunks.append({
                "chunk_id": str(uuid.uuid4()),
                "doc_id": doc_id,
                "tenant_id": tenant_id,
                "section": current_section,
                "content": chunk_text,
                "is_table": True,
                "page_number": page_no,
                "bbox": {"x0": bbox.l, "y0": bbox.t, "x1": bbox.r, "y1": bbox.b} if bbox else {}
            })
            
        elif item.label in ["paragraph", "text"] and len(item.text.strip()) > 40:
            chunk_text = f"Context: {current_section}\n\n{item.text.strip()}"
            bbox = item.prov[0].bbox if item.prov else None
            page_no = item.prov[0].page_no if item.prov else 1
            
            chunks.append({
                "chunk_id": str(uuid.uuid4()),
                "doc_id": doc_id,
                "tenant_id": tenant_id,
                "section": current_section,
                "content": chunk_text,
                "is_table": False,
                "page_number": page_no,
                "bbox": {"x0": bbox.l, "y0": bbox.t, "x1": bbox.r, "y1": bbox.b} if bbox else {}
            })
            
    return chunks
def batch_embed_chunks(chunks: list[dict]) -> list[list[float]]:
    """
    Calls local Hugging Face Text Embeddings Inference (TEI) service.
    Zero egress: traffic stays on the local 10.100.10.x subnet.
    """
    texts = [c["content"] for c in chunks]
    response = requests.post(
        TEI_EMBED_URL,
        json={"inputs": texts, "truncate": True},
        timeout=15
    )
    response.raise_for_status()
    return response.json()
def persist_to_pgvector(chunks: list[dict], embeddings: list[list[float]]):
    """
    Atomic multi-row insertion into PostgreSQL 16 with pgvector.
    Updates full-text tsvector automatically via SQL trigger.
    """
    records = []
    for c, emb in zip(chunks, embeddings):
        records.append((
            c["chunk_id"],
            c["doc_id"],
            c["tenant_id"],
            c["section"],
            c["content"],
            c["is_table"],
            c["page_number"],
            json.dumps(c["bbox"]),
            emb
        ))
        
    query = """
    INSERT INTO document_chunks (
        id, doc_id, tenant_id, section_name, content, 
        is_table, page_number, bbox_metadata, embedding
    ) VALUES %s
    ON CONFLICT (id) DO NOTHING;
    """
    
    with psycopg2.connect(PG_DSN) as conn:
        with conn.cursor() as cur:
            execute_values(cur, query, records, template="(%s, %s, %s, %s, %s, %s, %s, %s, %s::vector)")
        conn.commit()

flowchart LR
    subgraph Ingestion_Stage ["1. Layout Ingestion Pipeline"]
        PDF["Raw Financial PDF / Invoice"] --> Parser["Layout-Aware Parser<br/>(Table Grid Extraction)"]
        Parser --> Chunker["Semantic Hierarchy Chunking<br/>(Markdown Tables + Breadcrumbs)"]
        Chunker --> LocalEmbed["Local BGE-M3 Service<br/>(Dense 1024-dim Vector)"]
        Chunker --> TextIndex["Postgres tsvector<br/>(Sparse BM25 Tokens)"]
    end
    subgraph Storage_Stage ["2. Unified pgvector Storage"]
        LocalEmbed --> PG_HNSW[("pgvector HNSW Index<br/>(Cosine Distance)")]
        TextIndex --> PG_GIN[("PostgreSQL GIN Index<br/>(Lexical Matching)")]
    end
    subgraph Retrieval_Stage ["3. Hybrid Search & Reranking"]
        Query["Enterprise User Query"] --> DenseSearch["Dense HNSW Search<br/>(Top 50 Candidates)"]
        Query --> SparseSearch["BM25 Lexical Search<br/>(Top 50 Candidates)"]
        PG_HNSW -.-> DenseSearch
        PG_GIN -.-> SparseSearch
        DenseSearch --> RRF["Reciprocal Rank Fusion (RRF)<br/>Score = 1/(60 + Rank_Dense) + 1/(60 + Rank_Sparse)"]
        SparseSearch --> RRF
        RRF --> TopPool["Top 30 Merged Candidates"]
        TopPool --> Reranker["Local bge-reranker-large<br/>(Cross-Encoder Scoring)"]
        Reranker --> TopK["Top 5 High-Precision Chunks"]
    end
    subgraph Generation_Stage ["4. Sovereign Synthesis"]
        TopK --> ContextAssembler["Strict Prompt Assembler<br/>(Bounding Box Citations)"]
        ContextAssembler --> LocalLLM["Air-Gapped vLLM<br/>(Llama-3.1-70B-AWQ)"]
        LocalLLM --> VerifiedOutput["Grounded JSON Output<br/>+ Page Coordinate Highlights"]
    end

+─────────────────────────────────────────────────────────────────────────────────────────────────+
|               DETERMINISTIC INGESTION & HYBRID RETRIEVAL PIPELINE (ZERO EGRESS)                 |
+─────────────────────────────────────────────────────────────────────────────────────────────────+
|                                                                                                 |
| [1. Ingestion]                                                                                  |
|   PDF Invoice ──► Layout-Aware Parser ──► Markdown Tables ──► Parent-Child Chunks               |
|                                                                 │                               |
|                                ┌────────────────────────────────┴──────────────────────────┐    |
|                                ▼ (1024-dim Embedding)                                      ▼    |
|                     [Local BGE-M3 Engine]                               [tsvector Generator]    |
|                                │                                                   │            |
| [2. Storage]                   ▼                                                   ▼            |
|                     PostgreSQL pgvector (HNSW)                          PostgreSQL GIN (BM25)   |
|                                │                                                   │            |
|                                └────────────────────────────────┬──────────────────┘            |
|                                                                 │                               |
| [3. Retrieval] Enterprise Query ────────────────────────────────┼────────────────────────┐      |
|                                                                 ▼                        ▼      |
|                                                     Dense Top-50 Search         Sparse Top-50   |
|                                                                 │                        │      |
|                                                                 └───────────┬────────────┘      |
|                                                                             ▼                   |
|                                                               Reciprocal Rank Fusion (RRF)      |
|                                                                             │                   |
|                                                                             ▼                   |
|                                                               Local Cross-Encoder Reranker      |
|                                                               (bge-reranker-large: Top 5)       |
|                                                                             │                   |
| [4. Generation]                                                             ▼                   |
|   Air-Gapped vLLM (Llama-3.1-70B-AWQ) ◄── Grounded Prompt ◄── Context Assembly & Citations     |
|             │                                                                                   |
|             ▼                                                                                   |
|   Deterministic Output + Exact Page Bounding Box Highlights                                     |
+─────────────────────────────────────────────────────────────────────────────────────────────────+

-- Enable required extensions
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
CREATE EXTENSION IF NOT EXISTS "vector";
-- Production chunks storage table
CREATE TABLE document_chunks (
    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
    doc_id UUID NOT NULL,
    tenant_id VARCHAR(64) NOT NULL,
    clearance_level VARCHAR(32) NOT NULL DEFAULT 'standard',
    section_name VARCHAR(255) NOT NULL,
    content TEXT NOT NULL,
    content_tsv TSVECTOR GENERATED ALWAYS AS (to_tsvector('english', content)) STORED,
    is_table BOOLEAN NOT NULL DEFAULT FALSE,
    page_number INTEGER NOT NULL,
    bbox_metadata JSONB NOT NULL DEFAULT '{}'::jsonb,
    embedding VECTOR(1024) NOT NULL,
    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
-- HNSW Vector Index for dense cosine similarity
-- m = 16 (bi-directional links per node), ef_construction = 128 (graph build quality)
CREATE INDEX idx_chunks_hnsw_embedding 
ON document_chunks 
USING hnsw (embedding vector_cosine_ops)
WITH (m = 16, ef_construction = 128);
-- GIN Index for BM25-equivalent sparse full-text search
CREATE INDEX idx_chunks_content_tsv 
ON document_chunks 
USING gin (content_tsv);
-- B-Tree Index for multi-tenant isolation and security filtering
CREATE INDEX idx_chunks_tenant_clearance 
ON document_chunks (tenant_id, clearance_level);

WITH 
-- 1. Dense Semantic Vector Retrieval (Top 50)
dense_results AS (
    SELECT 
        id, 
        ROW_NUMBER() OVER (ORDER BY embedding <=> :query_vector) AS dense_rank
    FROM document_chunks
    WHERE tenant_id = :tenant_id
      AND clearance_level <= :user_clearance
    ORDER BY embedding <=> :query_vector
    LIMIT 50
),
-- 2. Sparse Lexical Full-Text Retrieval (Top 50)
sparse_results AS (
    SELECT 
        id, 
        ROW_NUMBER() OVER (ORDER BY ts_rank_cd(content_tsv, plainto_tsquery('english', :query_text)) DESC) AS sparse_rank
    FROM document_chunks
    WHERE tenant_id = :tenant_id
      AND clearance_level <= :user_clearance
      AND content_tsv @@ plainto_tsquery('english', :query_text)
    ORDER BY ts_rank_cd(content_tsv, plainto_tsquery('english', :query_text)) DESC
    LIMIT 50
)
-- 3. Reciprocal Rank Fusion Merge
SELECT 
    c.id,
    c.section_name,
    c.content,
    c.is_table,
    c.page_number,
    c.bbox_metadata,
    COALESCE(1.0 / (60 + d.dense_rank), 0.0) + 
    COALESCE(1.0 / (60 + s.sparse_rank), 0.0) AS rrf_score
FROM dense_results d
FULL OUTER JOIN sparse_results s ON d.id = s.id
JOIN document_chunks c ON c.id = COALESCE(d.id, s.id)
ORDER BY rrf_score DESC
LIMIT 30;

version: "3.8"
services:
  tei-embeddings:
    image: ghcr.io/huggingface/text-embeddings-inference:turing-1.5
    container_name: tei-bge-m3
    restart: always
    environment:
MODEL_ID=BAAI/bge-m3
MAX_CLIENT_BATCH_SIZE=64
MAX_BATCH_TOKENS=16384
    volumes:
/opt/models/bge-m3:/data
    deploy:
      resources:
        reservations:
          devices:
driver: nvidia
              count: 1
              capabilities: [gpu]
    networks:
vpc_internal
  vllm-inference:
    image: vllm/vllm-openai:v0.5.4
    container_name: vllm-llama3-70b
    restart: always
    environment:
NCCL_DEBUG=INFO
HF_HUB_OFFLINE=1  # Strictly enforces zero external model repository pulls
    command: >
      --model /opt/models/Meta-Llama-3.1-70B-Instruct-AWQ
      --quantization awq
      --tensor-parallel-size 2
      --max-model-len 8192
      --gpu-memory-utilization 0.92
      --enforce-eager
      --port 8000
    volumes:
/opt/models/Meta-Llama-3.1-70B-Instruct-AWQ:/opt/models/Meta-Llama-3.1-70B-Instruct-AWQ:ro
    deploy:
      resources:
        reservations:
          devices:
driver: nvidia
              count: 2
              capabilities: [gpu]
    networks:
vpc_internal
networks:
  vpc_internal:
    driver: bridge
    internal: true  # Prevents external network routing

-- Enable Row-Level Security on document chunks
ALTER TABLE document_chunks ENABLE ROW LEVEL SECURITY;
-- Create policy matching user's tenant ID and clearance level from session variables
CREATE POLICY tenant_isolation_policy ON document_chunks
    FOR SELECT
    USING (
        tenant_id = current_setting('app.current_tenant_id', true)
        AND (
            clearance_level = 'public'
            OR (clearance_level = 'internal' AND current_setting('app.user_role', true) IN ('employee', 'manager', 'admin'))
            OR (clearance_level = 'confidential' AND current_setting('app.user_role', true) IN ('manager', 'admin'))
            OR (clearance_level = 'restricted' AND current_setting('app.user_role', true) = 'admin')
        )
    );

def query_with_rls(tenant_id: str, user_role: str, query_vector: list[float], query_text: str):
    with db_pool.get_connection() as conn:
        with conn.cursor() as cur:
            # Inject session security variables for current transaction only
            cur.execute("SET LOCAL app.current_tenant_id = %s;", (tenant_id,))
            cur.execute("SET LOCAL app.user_role = %s;", (user_role,))
            
            # Execute hybrid search with 100% RLS policy enforcement
            cur.execute(HYBRID_SEARCH_SQL, {"query_vector": query_vector, "query_text": query_text})
            results = cur.fetchall()
            return results

  # 1. Audit active network routing table (Ensure no default 0.0.0.0/0 route exists)
ip route show
  # 2. Monitor for any unauthorized egress connection attempts across network interfaces
sudo tcpdump -i any -n "not (src net 10.100.0.0/16 and dst net 10.100.0.0/16)" -c 50
  # 3. Verify pgvector HNSW index health and memory footprint
psql "$PG_DSN" -c "
SELECT 
    schemaname, tablename, indexname, 
    pg_size_pretty(pg_relation_size(indexrelid)) AS index_size
FROM pg_stat_user_indexes 
WHERE indexname LIKE '%hnsw%';
"
  # 4. Probe local vLLM health and GPU memory allocation
curl -s http://10.100.10.25:8000/health | jq .
nvidia-smi --query-gpu=memory.total,memory.used,memory.free,utilization.gpu --format=csv