mirror of
https://github.com/kbenestad/mdcms.git
synced 2026-06-18 15:24:32 +00:00
Add neuraldb-docs site files (batch 1: config, nav, theme, pages)
This commit is contained in:
parent
a49fe4832e
commit
3058dbee3e
11 changed files with 1756 additions and 0 deletions
7
neuraldb-docs/config.yml
Normal file
7
neuraldb-docs/config.yml
Normal file
|
|
@ -0,0 +1,7 @@
|
||||||
|
# mdcms v0.3 | DO NOT REMOVE THIS COMMENT
|
||||||
|
sitename: NeuralDB
|
||||||
|
sitedescription: AI-native database with vector and relational capabilities
|
||||||
|
navigation: sidebar
|
||||||
|
search: true
|
||||||
|
footer: "© 2026 NeuralDB, Inc. Documentation licensed under CC BY 4.0."
|
||||||
|
theme: theme.yml
|
||||||
240
neuraldb-docs/nav.yml
Normal file
240
neuraldb-docs/nav.yml
Normal file
|
|
@ -0,0 +1,240 @@
|
||||||
|
# nav.yml — generated by mdcms.py
|
||||||
|
sections:
|
||||||
|
- code: overview
|
||||||
|
defaultname: Overview
|
||||||
|
sort: 100
|
||||||
|
pagesvisibility: visible
|
||||||
|
|
||||||
|
- code: installation
|
||||||
|
defaultname: Installation
|
||||||
|
sort: 200
|
||||||
|
pagesvisibility: visible
|
||||||
|
|
||||||
|
- code: configuration
|
||||||
|
defaultname: Configuration
|
||||||
|
sort: 300
|
||||||
|
pagesvisibility: visible
|
||||||
|
|
||||||
|
- code: query-language
|
||||||
|
defaultname: Query Language
|
||||||
|
sort: 400
|
||||||
|
pagesvisibility: visible
|
||||||
|
|
||||||
|
- code: client-sdks
|
||||||
|
defaultname: Client SDKs
|
||||||
|
sort: 500
|
||||||
|
pagesvisibility: visible
|
||||||
|
|
||||||
|
- code: operations
|
||||||
|
defaultname: Operations
|
||||||
|
sort: 600
|
||||||
|
pagesvisibility: visible
|
||||||
|
|
||||||
|
pages:
|
||||||
|
- file: pages/index.md
|
||||||
|
title: What is NeuralDB?
|
||||||
|
section-id: overview
|
||||||
|
sort: 100
|
||||||
|
variants: [en]
|
||||||
|
titles:
|
||||||
|
en: What is NeuralDB?
|
||||||
|
|
||||||
|
- file: pages/concepts.md
|
||||||
|
title: Core Concepts
|
||||||
|
section-id: overview
|
||||||
|
sort: 110
|
||||||
|
variants: [en]
|
||||||
|
titles:
|
||||||
|
en: Core Concepts
|
||||||
|
|
||||||
|
- file: pages/architecture.md
|
||||||
|
title: Architecture
|
||||||
|
section-id: overview
|
||||||
|
sort: 120
|
||||||
|
variants: [en]
|
||||||
|
titles:
|
||||||
|
en: Architecture
|
||||||
|
|
||||||
|
- file: pages/comparison.md
|
||||||
|
title: Comparison
|
||||||
|
section-id: overview
|
||||||
|
sort: 130
|
||||||
|
variants: [en]
|
||||||
|
titles:
|
||||||
|
en: Comparison
|
||||||
|
|
||||||
|
- file: pages/install-docker.md
|
||||||
|
title: Docker Install
|
||||||
|
section-id: installation
|
||||||
|
sort: 100
|
||||||
|
variants: [en]
|
||||||
|
titles:
|
||||||
|
en: Docker Install
|
||||||
|
|
||||||
|
- file: pages/install-kubernetes.md
|
||||||
|
title: Kubernetes
|
||||||
|
section-id: installation
|
||||||
|
sort: 110
|
||||||
|
variants: [en]
|
||||||
|
titles:
|
||||||
|
en: Kubernetes
|
||||||
|
|
||||||
|
- file: pages/install-cloud.md
|
||||||
|
title: Cloud Managed
|
||||||
|
section-id: installation
|
||||||
|
sort: 120
|
||||||
|
variants: [en]
|
||||||
|
titles:
|
||||||
|
en: Cloud Managed
|
||||||
|
|
||||||
|
- file: pages/install-local.md
|
||||||
|
title: Local Development
|
||||||
|
section-id: installation
|
||||||
|
sort: 130
|
||||||
|
variants: [en]
|
||||||
|
titles:
|
||||||
|
en: Local Development
|
||||||
|
|
||||||
|
- file: pages/config-server.md
|
||||||
|
title: Server Config
|
||||||
|
section-id: configuration
|
||||||
|
sort: 100
|
||||||
|
variants: [en]
|
||||||
|
titles:
|
||||||
|
en: Server Config
|
||||||
|
|
||||||
|
- file: pages/config-auth.md
|
||||||
|
title: Authentication
|
||||||
|
section-id: configuration
|
||||||
|
sort: 110
|
||||||
|
variants: [en]
|
||||||
|
titles:
|
||||||
|
en: Authentication
|
||||||
|
|
||||||
|
- file: pages/config-storage.md
|
||||||
|
title: Storage Config
|
||||||
|
section-id: configuration
|
||||||
|
sort: 120
|
||||||
|
variants: [en]
|
||||||
|
titles:
|
||||||
|
en: Storage Config
|
||||||
|
|
||||||
|
- file: pages/config-replication.md
|
||||||
|
title: Replication
|
||||||
|
section-id: configuration
|
||||||
|
sort: 130
|
||||||
|
variants: [en]
|
||||||
|
titles:
|
||||||
|
en: Replication
|
||||||
|
|
||||||
|
- file: pages/nql-basics.md
|
||||||
|
title: NQL Basics
|
||||||
|
section-id: query-language
|
||||||
|
sort: 100
|
||||||
|
variants: [en]
|
||||||
|
titles:
|
||||||
|
en: NQL Basics
|
||||||
|
|
||||||
|
- file: pages/nql-vectors.md
|
||||||
|
title: Vector Queries
|
||||||
|
section-id: query-language
|
||||||
|
sort: 110
|
||||||
|
variants: [en]
|
||||||
|
titles:
|
||||||
|
en: Vector Queries
|
||||||
|
|
||||||
|
- file: pages/nql-hybrid.md
|
||||||
|
title: Hybrid Queries
|
||||||
|
section-id: query-language
|
||||||
|
sort: 120
|
||||||
|
variants: [en]
|
||||||
|
titles:
|
||||||
|
en: Hybrid Queries
|
||||||
|
|
||||||
|
- file: pages/nql-aggregations.md
|
||||||
|
title: Aggregations
|
||||||
|
section-id: query-language
|
||||||
|
sort: 130
|
||||||
|
variants: [en]
|
||||||
|
titles:
|
||||||
|
en: Aggregations
|
||||||
|
|
||||||
|
- file: pages/nql-transactions.md
|
||||||
|
title: Transactions
|
||||||
|
section-id: query-language
|
||||||
|
sort: 140
|
||||||
|
variants: [en]
|
||||||
|
titles:
|
||||||
|
en: Transactions
|
||||||
|
|
||||||
|
- file: pages/sdk-python.md
|
||||||
|
title: Python SDK
|
||||||
|
section-id: client-sdks
|
||||||
|
sort: 100
|
||||||
|
variants: [en]
|
||||||
|
titles:
|
||||||
|
en: Python SDK
|
||||||
|
|
||||||
|
- file: pages/sdk-javascript.md
|
||||||
|
title: JavaScript SDK
|
||||||
|
section-id: client-sdks
|
||||||
|
sort: 110
|
||||||
|
variants: [en]
|
||||||
|
titles:
|
||||||
|
en: JavaScript SDK
|
||||||
|
|
||||||
|
- file: pages/sdk-go.md
|
||||||
|
title: Go SDK
|
||||||
|
section-id: client-sdks
|
||||||
|
sort: 120
|
||||||
|
variants: [en]
|
||||||
|
titles:
|
||||||
|
en: Go SDK
|
||||||
|
|
||||||
|
- file: pages/sdk-rest.md
|
||||||
|
title: REST API
|
||||||
|
section-id: client-sdks
|
||||||
|
sort: 130
|
||||||
|
variants: [en]
|
||||||
|
titles:
|
||||||
|
en: REST API
|
||||||
|
|
||||||
|
- file: pages/ops-monitoring.md
|
||||||
|
title: Monitoring
|
||||||
|
section-id: operations
|
||||||
|
sort: 100
|
||||||
|
variants: [en]
|
||||||
|
titles:
|
||||||
|
en: Monitoring
|
||||||
|
|
||||||
|
- file: pages/ops-backup.md
|
||||||
|
title: Backup & Restore
|
||||||
|
section-id: operations
|
||||||
|
sort: 110
|
||||||
|
variants: [en]
|
||||||
|
titles:
|
||||||
|
en: Backup & Restore
|
||||||
|
|
||||||
|
- file: pages/ops-scaling.md
|
||||||
|
title: Scaling
|
||||||
|
section-id: operations
|
||||||
|
sort: 120
|
||||||
|
variants: [en]
|
||||||
|
titles:
|
||||||
|
en: Scaling
|
||||||
|
|
||||||
|
- file: pages/ops-migration.md
|
||||||
|
title: Migration
|
||||||
|
section-id: operations
|
||||||
|
sort: 130
|
||||||
|
variants: [en]
|
||||||
|
titles:
|
||||||
|
en: Migration
|
||||||
|
|
||||||
|
- file: pages/ops-troubleshooting.md
|
||||||
|
title: Troubleshooting
|
||||||
|
section-id: operations
|
||||||
|
sort: 140
|
||||||
|
variants: [en]
|
||||||
|
titles:
|
||||||
|
en: Troubleshooting
|
||||||
188
neuraldb-docs/pages/architecture.md
Normal file
188
neuraldb-docs/pages/architecture.md
Normal file
|
|
@ -0,0 +1,188 @@
|
||||||
|
---
|
||||||
|
title: Architecture
|
||||||
|
sort: 120
|
||||||
|
section-id: overview
|
||||||
|
keywords: architecture, storage engine, query planner, replication, WAL, HNSW
|
||||||
|
description: NeuralDB internal architecture — storage engine, query planner, and replication
|
||||||
|
language: en
|
||||||
|
---
|
||||||
|
|
||||||
|
# Architecture
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
NeuralDB is built on a custom storage engine that co-locates relational and vector data, with a query planner that understands both SQL predicates and vector similarity operations natively.
|
||||||
|
|
||||||
|
## High-Level Architecture
|
||||||
|
|
||||||
|
```
|
||||||
|
Client (psql / SDK / REST)
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
┌─────────────────────────────────────────┐
|
||||||
|
│ Connection Layer │
|
||||||
|
│ (PostgreSQL Wire Protocol compatible) │
|
||||||
|
└───────────────────┤─────────────────────┘
|
||||||
|
│
|
||||||
|
┌──────────▼──────────┐
|
||||||
|
│ Query Parser │
|
||||||
|
│ (SQL + NQL ext.) │
|
||||||
|
└──────────┬──────────┘
|
||||||
|
│
|
||||||
|
┌──────────▼──────────┐
|
||||||
|
│ Semantic Planner │◄── Statistics + Index Metadata
|
||||||
|
│ (hybrid cost model) │
|
||||||
|
└──────────┬──────────┘
|
||||||
|
│
|
||||||
|
┌──────────▼──────────┐
|
||||||
|
│ Execution Engine │
|
||||||
|
│ ┌────────────────┐ │
|
||||||
|
│ │ SQL Executor │ │
|
||||||
|
│ └────────────────┘ │
|
||||||
|
│ ┌────────────────┐ │
|
||||||
|
│ │ ANN Executor │ │
|
||||||
|
│ └────────────────┘ │
|
||||||
|
└──────────┬──────────┘
|
||||||
|
│
|
||||||
|
┌──────────▼──────────┐
|
||||||
|
│ Storage Engine │
|
||||||
|
│ ┌────────────────┐ │
|
||||||
|
│ │ Row Store │ │◄── SST Files (columnar)
|
||||||
|
│ └────────────────┘ │
|
||||||
|
│ ┌────────────────┐ │
|
||||||
|
│ │ Vector Store │ │◄── HNSW Graph Files
|
||||||
|
│ └────────────────┘ │
|
||||||
|
│ ┌────────────────┐ │
|
||||||
|
│ │ WAL │ │◄── Write-Ahead Log
|
||||||
|
│ └────────────────┘ │
|
||||||
|
└─────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
## Storage Engine
|
||||||
|
|
||||||
|
### Row Store
|
||||||
|
|
||||||
|
NeuralDB's row store uses a Log-Structured Merge-tree (LSM) architecture inspired by RocksDB. Data is written to an in-memory write buffer (MemTable), which is periodically flushed to sorted string tables (SSTables) on disk. Background compaction merges SSTables and reclaims space.
|
||||||
|
|
||||||
|
Key properties:
|
||||||
|
- **Write-optimised**: writes are sequential, not random — excellent NVMe utilisation
|
||||||
|
- **Columnar format**: SSTables store data column-by-column for fast analytical scans
|
||||||
|
- **Compression**: LZ4 by default, Zstd for archival storage — typically 3–6× compression ratio
|
||||||
|
|
||||||
|
### Vector Store
|
||||||
|
|
||||||
|
Vectors are stored separately from rows in a Vector Store. The Vector Store maintains:
|
||||||
|
|
||||||
|
1. **Raw vector data** — the float32 arrays, stored in compressed pages
|
||||||
|
2. **HNSW graph** — the in-memory navigation graph for ANN search
|
||||||
|
|
||||||
|
The HNSW graph is loaded into memory on startup and kept warm. Memory required ≈ `num_vectors × dimensions × 4 bytes × 1.3` (1.3× overhead for the graph structure).
|
||||||
|
|
||||||
|
For a 10M-row table with 1536-dimensional embeddings: `10M × 1536 × 4 × 1.3 ≈ 80 GB`. Plan memory accordingly.
|
||||||
|
|
||||||
|
### Write-Ahead Log (WAL)
|
||||||
|
|
||||||
|
All writes (row and vector) are first written to the WAL before being applied to the storage engine. The WAL provides:
|
||||||
|
|
||||||
|
- **Durability**: committed transactions survive crashes
|
||||||
|
- **Replication**: replicas apply the WAL stream from the primary
|
||||||
|
- **Point-in-time recovery (PITR)**: archive the WAL to recover to any point in time
|
||||||
|
|
||||||
|
WAL segments are 128 MB by default and are archived to the configured storage backend (local disk, S3, GCS) upon rotation.
|
||||||
|
|
||||||
|
## Query Planner
|
||||||
|
|
||||||
|
The Semantic Planner extends a PostgreSQL-compatible query planner with understanding of vector operations.
|
||||||
|
|
||||||
|
### Hybrid Cost Model
|
||||||
|
|
||||||
|
For hybrid queries (vector + relational), the planner considers two physical plans:
|
||||||
|
|
||||||
|
**Plan A: Pre-filter**
|
||||||
|
```
|
||||||
|
Filter(price < 100) → ANN(embedding, k=10)
|
||||||
|
```
|
||||||
|
Cost: selectivity × full_scan_cost + ANN_cost(filtered_set)
|
||||||
|
|
||||||
|
**Plan B: Post-filter**
|
||||||
|
```
|
||||||
|
ANN(embedding, k=10×estimated_filter_ratio) → Filter(price < 100)
|
||||||
|
```
|
||||||
|
Cost: ANN_cost(full_index) + filter_cost
|
||||||
|
|
||||||
|
The planner uses column statistics (histogram, null fraction, distinct values) and vector index parameters to estimate costs. It picks the plan with the lower estimated cost.
|
||||||
|
|
||||||
|
### Index Types
|
||||||
|
|
||||||
|
NeuralDB supports the following index types:
|
||||||
|
|
||||||
|
| Index | Data | Purpose |
|
||||||
|
|-------|------|-------|
|
||||||
|
| B-tree | Scalar columns | Equality, range queries |
|
||||||
|
| Hash | Scalar columns | Equality only (faster than B-tree) |
|
||||||
|
| GIN | JSON, arrays | Containment queries |
|
||||||
|
| HNSW | VECTOR columns | Approximate nearest neighbour |
|
||||||
|
| IVF-Flat | VECTOR columns | High-recall exact-ish search |
|
||||||
|
| BRIN | Timestamp columns | Range scans on append-only data |
|
||||||
|
|
||||||
|
## Replication
|
||||||
|
|
||||||
|
NeuralDB uses streaming replication. The primary continuously ships WAL segments to replicas, which apply them in order.
|
||||||
|
|
||||||
|
### Synchronous vs Asynchronous Replication
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- Set replication mode per-transaction
|
||||||
|
SET synchronous_commit = 'on'; -- wait for WAL to reach all sync replicas (safest)
|
||||||
|
SET synchronous_commit = 'local'; -- wait for local WAL flush only (faster)
|
||||||
|
SET synchronous_commit = 'off'; -- don't wait (fastest, small durability window)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Read Replicas
|
||||||
|
|
||||||
|
Replicas accept `SELECT` queries. Direct read-heavy workloads to replicas:
|
||||||
|
|
||||||
|
```
|
||||||
|
primary: write queries + critical reads
|
||||||
|
replica-1: analytical queries, reporting
|
||||||
|
replica-2: search API traffic
|
||||||
|
```
|
||||||
|
|
||||||
|
The client SDK supports automatic read/write splitting:
|
||||||
|
|
||||||
|
```python
|
||||||
|
client = NeuralDB(
|
||||||
|
primary="primary.example.com:5432",
|
||||||
|
replicas=["replica1.example.com:5432", "replica2.example.com:5432"],
|
||||||
|
read_from="replicas",
|
||||||
|
replica_selection="round-robin",
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Memory Architecture
|
||||||
|
|
||||||
|
NeuralDB divides available memory into three pools:
|
||||||
|
|
||||||
|
| Pool | Purpose | Default |
|
||||||
|
|------|---------|-------|
|
||||||
|
| `shared_buffers` | Row store page cache | 25% of RAM |
|
||||||
|
| `vector_buffer` | HNSW graph warm cache | 40% of RAM |
|
||||||
|
| `work_mem` | Per-query sort and hash buffers | 64 MB |
|
||||||
|
|
||||||
|
Tune these in `neuraldb.conf`:
|
||||||
|
|
||||||
|
```ini
|
||||||
|
shared_buffers = 8GB
|
||||||
|
vector_buffer = 16GB
|
||||||
|
work_mem = 128MB
|
||||||
|
```
|
||||||
|
|
||||||
|
## Consistency Model
|
||||||
|
|
||||||
|
NeuralDB provides **strong consistency** for primary reads and **eventual consistency** for replica reads (with a configurable replication lag threshold).
|
||||||
|
|
||||||
|
Reads on the primary always see the latest committed data. Reads on replicas may lag behind the primary by the `max_replication_lag` setting (default: 1 second). To force a replica to wait until it is caught up:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT pg_wait_for_replica_replay('0/1234ABCD');
|
||||||
|
```
|
||||||
102
neuraldb-docs/pages/comparison.md
Normal file
102
neuraldb-docs/pages/comparison.md
Normal file
|
|
@ -0,0 +1,102 @@
|
||||||
|
---
|
||||||
|
title: Comparison
|
||||||
|
sort: 130
|
||||||
|
section-id: overview
|
||||||
|
keywords: comparison, Postgres, pgvector, Pinecone, Weaviate, MongoDB, alternatives
|
||||||
|
description: How NeuralDB compares to Postgres+pgvector, Pinecone, Weaviate, and MongoDB Atlas Vector Search
|
||||||
|
language: en
|
||||||
|
---
|
||||||
|
|
||||||
|
# Comparison
|
||||||
|
|
||||||
|
This page provides an honest comparison of NeuralDB against the most common alternatives for applications that need vector search.
|
||||||
|
|
||||||
|
## NeuralDB vs PostgreSQL + pgvector
|
||||||
|
|
||||||
|
pgvector is the most popular way to add vector search to an existing PostgreSQL deployment. If you already run Postgres, the low barrier to entry is attractive. Here is where the two diverge:
|
||||||
|
|
||||||
|
| Feature | NeuralDB | Postgres + pgvector |
|
||||||
|
|---------|---------|---------------------|
|
||||||
|
| Vector index algorithm | HNSW (native) | HNSW, IVFFlat |
|
||||||
|
| Max dimensions | 65,536 | 16,000 |
|
||||||
|
| Index build speed | Native Rust (fast) | C extension (moderate) |
|
||||||
|
| Parallel index builds | Yes | Limited |
|
||||||
|
| Vector data memory isolation | Dedicated vector buffer pool | Shared with row pages |
|
||||||
|
| Horizontal sharding | Built-in | Manual (Citus, Patroni) |
|
||||||
|
| Automatic embeddings | Yes | No |
|
||||||
|
| Multi-modal vectors | Yes | No (one VECTOR column type) |
|
||||||
|
| Streaming ingestion | Yes | No |
|
||||||
|
| Read replica for vector queries | Automatic routing | Manual |
|
||||||
|
|
||||||
|
**When to choose Postgres + pgvector:** You already have a Postgres deployment, your dataset is under 10M vectors, and you do not need automatic embeddings or horizontal scaling. The operational overhead of a new database system is not worth it.
|
||||||
|
|
||||||
|
**When to choose NeuralDB:** Your vector dataset exceeds 10M rows, you need horizontal sharding, you want automatic embedding pipelines, or you are starting a new project and want a purpose-built system.
|
||||||
|
|
||||||
|
## NeuralDB vs Pinecone
|
||||||
|
|
||||||
|
Pinecone is a fully managed, purpose-built vector database. It excels at pure vector search at massive scale.
|
||||||
|
|
||||||
|
| Feature | NeuralDB | Pinecone |
|
||||||
|
|---------|---------|-------|
|
||||||
|
| Relational data | Full SQL | Metadata filters only |
|
||||||
|
| Hybrid queries | Single query, query planner | Metadata post-filter |
|
||||||
|
| ACID transactions | Yes | No |
|
||||||
|
| SQL interface | Yes | Proprietary API |
|
||||||
|
| Self-hosted option | Yes | No |
|
||||||
|
| Pricing model | Infrastructure cost | Per-request + storage |
|
||||||
|
| Latency (p99, 1M vectors) | ~5ms | ~10ms (managed) |
|
||||||
|
| Data gravity | Stays in your infra | Vendor-managed |
|
||||||
|
|
||||||
|
**When to choose Pinecone:** You need a fully managed solution with no operational overhead, your workload is pure vector search with simple metadata filtering, and you are comfortable with a vendor-specific API and pricing model.
|
||||||
|
|
||||||
|
**When to choose NeuralDB:** You need relational data co-located with vectors, ACID transactions, SQL compatibility, self-hosting, or lower total cost of ownership at scale.
|
||||||
|
|
||||||
|
## NeuralDB vs Weaviate
|
||||||
|
|
||||||
|
Weaviate is an open-source vector database with a GraphQL-based query language and built-in module support for embedding generation.
|
||||||
|
|
||||||
|
| Feature | NeuralDB | Weaviate |
|
||||||
|
|---------|---------|-------|
|
||||||
|
| Query language | SQL (NQL) | GraphQL |
|
||||||
|
| Relational joins | Yes | No |
|
||||||
|
| ACID transactions | Yes | Eventually consistent |
|
||||||
|
| SQL wire compatibility | PostgreSQL wire protocol | Proprietary |
|
||||||
|
| Embedding modules | Yes | Yes (vectorizers) |
|
||||||
|
| BM25 hybrid search | Yes | Yes |
|
||||||
|
| Multi-tenancy | Row-level, schema-level | Class-level |
|
||||||
|
| Replication | Sync + async | Eventual |
|
||||||
|
|
||||||
|
**When to choose Weaviate:** You want an open-source solution with a rich ecosystem of vectorizer modules and a GraphQL interface. If your team is more comfortable with graph-shaped queries than SQL, Weaviate is a natural fit.
|
||||||
|
|
||||||
|
**When to choose NeuralDB:** You need SQL, transactional guarantees, relational joins between your vector data and other structured data, or PostgreSQL wire protocol compatibility (so existing tools like dbt, Metabase, and psql work out of the box).
|
||||||
|
|
||||||
|
## NeuralDB vs MongoDB Atlas Vector Search
|
||||||
|
|
||||||
|
MongoDB Atlas added vector search as an extension to its document model. It is a convenient choice if you already run Atlas.
|
||||||
|
|
||||||
|
| Feature | NeuralDB | MongoDB Atlas Vector Search |
|
||||||
|
|---------|---------|---------------------------|
|
||||||
|
| Data model | Relational + vector | Document + vector |
|
||||||
|
| Query language | SQL | MQL (MongoDB Query Language) |
|
||||||
|
| ACID transactions | Yes (all operations) | Yes (within a session) |
|
||||||
|
| Horizontal scaling | Native sharding | Atlas sharding |
|
||||||
|
| Vector index type | HNSW | ENN (exact), HNSW |
|
||||||
|
| Full-text + vector hybrid | Yes | Yes (Atlas Search) |
|
||||||
|
| Self-hosted | Yes | Atlas only |
|
||||||
|
|
||||||
|
**When to choose MongoDB Atlas Vector Search:** Your application already uses MongoDB and you want to add vector search without changing your data model or infrastructure. The document model maps well to semi-structured data.
|
||||||
|
|
||||||
|
**When to choose NeuralDB:** You need relational data integrity, SQL, lower query latency, or the ability to self-host. If your data is inherently tabular (rather than document-shaped), NeuralDB's relational model will be a better fit.
|
||||||
|
|
||||||
|
## Performance Benchmarks
|
||||||
|
|
||||||
|
The following benchmarks were run against 10M 1536-dimensional vectors on equivalent hardware (32 vCPU, 128 GB RAM, NVMe SSD):
|
||||||
|
|
||||||
|
| System | QPS (recall@95%) | p50 latency | p99 latency | Index build time |
|
||||||
|
|--------|-----------------|-------------|-------------|----------------|
|
||||||
|
| NeuralDB 1.0 | 8,400 | 1.2ms | 4.8ms | 22 min |
|
||||||
|
| pgvector 0.7 | 3,100 | 2.9ms | 12ms | 45 min |
|
||||||
|
| Pinecone (s1) | 5,200 | 1.8ms | 8ms | Managed |
|
||||||
|
| Weaviate 1.24 | 4,600 | 2.1ms | 9ms | 31 min |
|
||||||
|
|
||||||
|
Benchmarks are inherently workload-dependent. Run your own benchmarks against your specific data and query patterns before making infrastructure decisions.
|
||||||
171
neuraldb-docs/pages/concepts.md
Normal file
171
neuraldb-docs/pages/concepts.md
Normal file
|
|
@ -0,0 +1,171 @@
|
||||||
|
---
|
||||||
|
title: Core Concepts
|
||||||
|
sort: 110
|
||||||
|
section-id: overview
|
||||||
|
keywords: concepts, vectors, embeddings, hybrid queries, nodes, HNSW, ANN
|
||||||
|
description: Core NeuralDB concepts — vectors, embeddings, hybrid queries, and the node model
|
||||||
|
language: en
|
||||||
|
---
|
||||||
|
|
||||||
|
# Core Concepts
|
||||||
|
|
||||||
|
Understanding these fundamental concepts will help you use NeuralDB effectively and make good architectural decisions for your application.
|
||||||
|
|
||||||
|
## Vectors and Embeddings
|
||||||
|
|
||||||
|
A **vector** is an ordered list of floating-point numbers — a point in high-dimensional space. In NeuralDB, vectors are used to represent the semantic meaning of data.
|
||||||
|
|
||||||
|
An **embedding** is a vector produced by a machine learning model that encodes the semantic meaning of its input. Similar inputs produce vectors that are close together in the embedding space. For example, the sentences "I love dogs" and "I adore canines" will produce embeddings that are close to each other, even though they share no words.
|
||||||
|
|
||||||
|
NeuralDB stores embeddings as `VECTOR(n)` columns, where `n` is the dimensionality (the number of float32 values). Common dimensionalities:
|
||||||
|
|
||||||
|
| Model | Dimensions |
|
||||||
|
|-------|----------|
|
||||||
|
| OpenAI text-embedding-3-small | 1536 |
|
||||||
|
| OpenAI text-embedding-3-large | 3072 |
|
||||||
|
| Cohere embed-english-v3.0 | 1024 |
|
||||||
|
| Google text-embedding-004 | 768 |
|
||||||
|
| BAAI/bge-m3 | 1024 |
|
||||||
|
|
||||||
|
## Distance Metrics
|
||||||
|
|
||||||
|
NeuralDB computes similarity between two vectors using one of three distance metrics:
|
||||||
|
|
||||||
|
### Cosine Similarity
|
||||||
|
|
||||||
|
Measures the angle between two vectors. Ranges from -1 (opposite) to 1 (identical). Ideal for text embeddings produced by models trained with cosine objectives:
|
||||||
|
|
||||||
|
```
|
||||||
|
cosine_similarity(a, b) = (a · b) / (|a| × |b|)
|
||||||
|
```
|
||||||
|
|
||||||
|
In NQL, use the `<=>` operator or `COSINE_SIMILARITY()` function.
|
||||||
|
|
||||||
|
### Dot Product
|
||||||
|
|
||||||
|
Measures the product of vector magnitudes and the angle between them. Used when vectors are not normalised and magnitude carries information (e.g., collaborative filtering):
|
||||||
|
|
||||||
|
```
|
||||||
|
dot_product(a, b) = Σ(aᵢ × bᵢ)
|
||||||
|
```
|
||||||
|
|
||||||
|
In NQL, use the `<#>` operator or `DOT_PRODUCT()` function.
|
||||||
|
|
||||||
|
### Euclidean Distance (L2)
|
||||||
|
|
||||||
|
Measures straight-line distance between two points. Lower is more similar. Useful for spatial data and image embeddings:
|
||||||
|
|
||||||
|
```
|
||||||
|
l2_distance(a, b) = √(Σ(aᵢ - bᵢ)²)
|
||||||
|
```
|
||||||
|
|
||||||
|
In NQL, use the `<->` operator or `L2_DISTANCE()` function.
|
||||||
|
|
||||||
|
## Vector Indexes
|
||||||
|
|
||||||
|
NeuralDB builds vector indexes using the **HNSW (Hierarchical Navigable Small World)** algorithm. HNSW provides:
|
||||||
|
|
||||||
|
- Sub-linear approximate nearest neighbour (ANN) search
|
||||||
|
- Configurable trade-off between speed and recall
|
||||||
|
- Incremental updates (no full rebuild needed when inserting)
|
||||||
|
|
||||||
|
### HNSW Parameters
|
||||||
|
|
||||||
|
When creating a vector index:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
CREATE INDEX ON documents
|
||||||
|
USING hnsw (embedding vector_cosine_ops)
|
||||||
|
WITH (m = 16, ef_construction = 64);
|
||||||
|
```
|
||||||
|
|
||||||
|
| Parameter | Description | Default |
|
||||||
|
|-----------|-------------|-------|
|
||||||
|
| `m` | Number of bi-directional links per node. Higher = better recall, more memory | 16 |
|
||||||
|
| `ef_construction` | Size of candidate set during index construction. Higher = better quality, slower build | 64 |
|
||||||
|
| `ef_search` | Size of candidate set at query time. Set per-query with `SET hnsw.ef_search = 100` | 40 |
|
||||||
|
|
||||||
|
### Exact vs Approximate Search
|
||||||
|
|
||||||
|
By default, vector queries use the HNSW index (approximate). For exact results (slower but 100% recall), use:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SET neuraldb.vector_scan = 'exact';
|
||||||
|
SELECT * FROM documents ORDER BY embedding <=> :query LIMIT 10;
|
||||||
|
```
|
||||||
|
|
||||||
|
## Hybrid Queries
|
||||||
|
|
||||||
|
A hybrid query combines vector similarity with relational predicates in a single query plan. The NeuralDB query planner evaluates two strategies and picks the cheaper one:
|
||||||
|
|
||||||
|
1. **Pre-filter then search** — apply relational filters first to reduce the candidate set, then run ANN search on the filtered set
|
||||||
|
2. **Post-filter** — run ANN search to get top-k candidates, then apply relational filters
|
||||||
|
|
||||||
|
NeuralDB automatically selects the optimal strategy based on selectivity estimates. You can hint the planner:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT * FROM documents
|
||||||
|
WHERE /*+ PREFILTER */ category = 'news'
|
||||||
|
ORDER BY embedding <=> :query
|
||||||
|
LIMIT 10;
|
||||||
|
```
|
||||||
|
|
||||||
|
## Tables and Schemas
|
||||||
|
|
||||||
|
NeuralDB is schema-based, like PostgreSQL. Everything lives inside a database → schema → table hierarchy:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
CREATE DATABASE my_app;
|
||||||
|
\c my_app
|
||||||
|
|
||||||
|
CREATE SCHEMA vectors;
|
||||||
|
CREATE SCHEMA metadata;
|
||||||
|
|
||||||
|
CREATE TABLE vectors.documents (
|
||||||
|
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||||
|
content TEXT NOT NULL,
|
||||||
|
embedding VECTOR(1536),
|
||||||
|
schema_id TEXT REFERENCES metadata.schemas(id),
|
||||||
|
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
||||||
|
);
|
||||||
|
```
|
||||||
|
|
||||||
|
## Nodes
|
||||||
|
|
||||||
|
NeuralDB is a distributed system. A **node** is a single NeuralDB process. Nodes have one of three roles:
|
||||||
|
|
||||||
|
| Role | Responsibilities |
|
||||||
|
|------|----------------|
|
||||||
|
| **Primary** | Accepts reads and writes, coordinates transactions |
|
||||||
|
| **Replica** | Accepts reads, replicates writes from primary |
|
||||||
|
| **Index** | Maintains vector indexes for shard(s), offloads ANN queries |
|
||||||
|
|
||||||
|
In a single-node deployment, one process takes all three roles.
|
||||||
|
|
||||||
|
## Sharding
|
||||||
|
|
||||||
|
NeuralDB shards data by `shard_key`. By default, the primary key is used as the shard key:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
CREATE TABLE events (
|
||||||
|
id UUID PRIMARY KEY,
|
||||||
|
tenant_id UUID NOT NULL,
|
||||||
|
event_type TEXT,
|
||||||
|
embedding VECTOR(768)
|
||||||
|
) SHARD BY tenant_id;
|
||||||
|
```
|
||||||
|
|
||||||
|
All rows with the same `tenant_id` are guaranteed to reside on the same shard, which enables efficient tenant-scoped queries without cross-shard joins.
|
||||||
|
|
||||||
|
## Transactions
|
||||||
|
|
||||||
|
NeuralDB uses multi-version concurrency control (MVCC) for transaction isolation, identical to PostgreSQL:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
BEGIN;
|
||||||
|
INSERT INTO documents (content, embedding) VALUES ($1, $2);
|
||||||
|
UPDATE document_counts SET count = count + 1 WHERE id = $3;
|
||||||
|
COMMIT;
|
||||||
|
```
|
||||||
|
|
||||||
|
Both the vector data and the relational data are committed atomically. If the transaction rolls back, neither the row nor the vector index entry is committed.
|
||||||
223
neuraldb-docs/pages/config-auth.md
Normal file
223
neuraldb-docs/pages/config-auth.md
Normal file
|
|
@ -0,0 +1,223 @@
|
||||||
|
---
|
||||||
|
title: Authentication
|
||||||
|
sort: 110
|
||||||
|
section-id: configuration
|
||||||
|
keywords: authentication, API keys, mTLS, RBAC, SSO, pg_hba, security
|
||||||
|
description: Configuring NeuralDB authentication — API keys, mTLS, role-based access control, and SSO
|
||||||
|
language: en
|
||||||
|
---
|
||||||
|
|
||||||
|
# Authentication
|
||||||
|
|
||||||
|
NeuralDB supports multiple authentication methods, from simple password auth to mutual TLS and enterprise SSO. This page explains how to configure each.
|
||||||
|
|
||||||
|
## Host-Based Authentication (pg_hba.conf)
|
||||||
|
|
||||||
|
The `pg_hba.conf` file controls which clients can connect and how they must authenticate. It is evaluated top-to-bottom and the first matching rule applies.
|
||||||
|
|
||||||
|
```
|
||||||
|
# pg_hba.conf
|
||||||
|
# FORMAT: TYPE DATABASE USER ADDRESS METHOD
|
||||||
|
|
||||||
|
# Local Unix socket connections (no password needed for postgres superuser)
|
||||||
|
local all neuraldb trust
|
||||||
|
local all all md5
|
||||||
|
|
||||||
|
# IPv4 connections from local network
|
||||||
|
host all all 127.0.0.1/32 scram-sha-256
|
||||||
|
host all all 10.0.0.0/8 scram-sha-256
|
||||||
|
|
||||||
|
# Reject all other connections
|
||||||
|
host all all 0.0.0.0/0 reject
|
||||||
|
```
|
||||||
|
|
||||||
|
Reload after changes:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT pg_reload_conf();
|
||||||
|
```
|
||||||
|
|
||||||
|
## Authentication Methods
|
||||||
|
|
||||||
|
| Method | Security | Use case |
|
||||||
|
|--------|---------|-------|
|
||||||
|
| `trust` | None | Local socket, trusted environments |
|
||||||
|
| `md5` | Weak | Legacy compatibility |
|
||||||
|
| `scram-sha-256` | Strong (default) | Standard password auth |
|
||||||
|
| `cert` | Very strong | Mutual TLS authentication |
|
||||||
|
| `ldap` | Strong | Enterprise LDAP/AD integration |
|
||||||
|
| `radius` | Strong | RADIUS server |
|
||||||
|
| `gss` | Strong | Kerberos |
|
||||||
|
| `jwt` | Strong | Token-based auth |
|
||||||
|
|
||||||
|
### Enabling scram-sha-256
|
||||||
|
|
||||||
|
```ini
|
||||||
|
# neuraldb.conf
|
||||||
|
password_encryption = scram-sha-256
|
||||||
|
```
|
||||||
|
|
||||||
|
Set passwords for users:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
CREATE USER app_user WITH PASSWORD 'secure-random-password';
|
||||||
|
ALTER USER app_user PASSWORD 'new-secure-password';
|
||||||
|
```
|
||||||
|
|
||||||
|
## Role-Based Access Control (RBAC)
|
||||||
|
|
||||||
|
### Creating Roles
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- Application read-only role
|
||||||
|
CREATE ROLE app_readonly;
|
||||||
|
GRANT CONNECT ON DATABASE myapp TO app_readonly;
|
||||||
|
GRANT USAGE ON SCHEMA public TO app_readonly;
|
||||||
|
GRANT SELECT ON ALL TABLES IN SCHEMA public TO app_readonly;
|
||||||
|
ALTER DEFAULT PRIVILEGES IN SCHEMA public
|
||||||
|
GRANT SELECT ON TABLES TO app_readonly;
|
||||||
|
|
||||||
|
-- Application read-write role
|
||||||
|
CREATE ROLE app_readwrite;
|
||||||
|
GRANT app_readonly TO app_readwrite;
|
||||||
|
GRANT INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO app_readwrite;
|
||||||
|
ALTER DEFAULT PRIVILEGES IN SCHEMA public
|
||||||
|
GRANT INSERT, UPDATE, DELETE ON TABLES TO app_readwrite;
|
||||||
|
|
||||||
|
-- Vector operations role (needed for ANN searches)
|
||||||
|
CREATE ROLE vector_user;
|
||||||
|
GRANT USAGE ON SCHEMA public TO vector_user;
|
||||||
|
GRANT SELECT ON ALL TABLES IN SCHEMA public TO vector_user;
|
||||||
|
GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO vector_user;
|
||||||
|
|
||||||
|
-- Application user
|
||||||
|
CREATE USER my_app WITH PASSWORD 'app-password';
|
||||||
|
GRANT app_readwrite TO my_app;
|
||||||
|
GRANT vector_user TO my_app;
|
||||||
|
```
|
||||||
|
|
||||||
|
### Row-Level Security (RLS)
|
||||||
|
|
||||||
|
For multi-tenant applications, use Row-Level Security to enforce tenant isolation at the database level:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
ALTER TABLE documents ENABLE ROW LEVEL SECURITY;
|
||||||
|
|
||||||
|
-- Tenants can only see their own documents
|
||||||
|
CREATE POLICY tenant_isolation ON documents
|
||||||
|
USING (tenant_id = current_setting('app.tenant_id')::UUID);
|
||||||
|
|
||||||
|
-- Admin can see everything
|
||||||
|
CREATE POLICY admin_access ON documents
|
||||||
|
TO admin_role
|
||||||
|
USING (true);
|
||||||
|
```
|
||||||
|
|
||||||
|
In your application, set the tenant context before querying:
|
||||||
|
|
||||||
|
```python
|
||||||
|
with connection.cursor() as cursor:
|
||||||
|
cursor.execute("SET app.tenant_id = %s", [tenant_id])
|
||||||
|
cursor.execute("SELECT * FROM documents ORDER BY embedding <=> %s LIMIT 10", [embedding])
|
||||||
|
```
|
||||||
|
|
||||||
|
## API Key Authentication
|
||||||
|
|
||||||
|
NeuralDB supports an API key table for token-based authentication — useful for microservices and CI pipelines that cannot use password auth:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- Enable the API key extension
|
||||||
|
CREATE EXTENSION neuraldb_apikeys;
|
||||||
|
|
||||||
|
-- Create an API key for a service account
|
||||||
|
SELECT neuraldb_apikeys.create_key(
|
||||||
|
label => 'my-service',
|
||||||
|
role => 'app_readwrite'
|
||||||
|
);
|
||||||
|
-- Returns: ndb_live_abcdefghij123456...
|
||||||
|
```
|
||||||
|
|
||||||
|
Clients authenticate by passing the key in the connection string:
|
||||||
|
|
||||||
|
```
|
||||||
|
postgresql://apikey:ndb_live_abc123@host:5432/mydb
|
||||||
|
```
|
||||||
|
|
||||||
|
Revoke a key:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT neuraldb_apikeys.revoke_key('ndb_live_abc123');
|
||||||
|
```
|
||||||
|
|
||||||
|
## Mutual TLS (mTLS)
|
||||||
|
|
||||||
|
For the highest security, require clients to present a certificate signed by your CA.
|
||||||
|
|
||||||
|
### 1. Generate a CA
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Generate CA key and certificate
|
||||||
|
openssl genrsa -out ca.key 4096
|
||||||
|
openssl req -new -x509 -key ca.key -out ca.crt -days 3650 \
|
||||||
|
-subj "/CN=NeuralDB CA/O=My Org"
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2. Issue a Server Certificate
|
||||||
|
|
||||||
|
```bash
|
||||||
|
openssl genrsa -out server.key 2048
|
||||||
|
openssl req -new -key server.key -out server.csr \
|
||||||
|
-subj "/CN=neuraldb.example.com"
|
||||||
|
openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key \
|
||||||
|
-CAcreateserial -out server.crt -days 365
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3. Configure NeuralDB
|
||||||
|
|
||||||
|
```ini
|
||||||
|
# neuraldb.conf
|
||||||
|
ssl = on
|
||||||
|
ssl_cert_file = '/etc/neuraldb/ssl/server.crt'
|
||||||
|
ssl_key_file = '/etc/neuraldb/ssl/server.key'
|
||||||
|
ssl_ca_file = '/etc/neuraldb/ssl/ca.crt'
|
||||||
|
ssl_crl_file = '/etc/neuraldb/ssl/crl.pem' # optional certificate revocation list
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4. Require Client Certificates
|
||||||
|
|
||||||
|
```
|
||||||
|
# pg_hba.conf
|
||||||
|
hostssl all all 0.0.0.0/0 cert clientcert=verify-full
|
||||||
|
```
|
||||||
|
|
||||||
|
## LDAP / Active Directory
|
||||||
|
|
||||||
|
```
|
||||||
|
# pg_hba.conf
|
||||||
|
host all all 0.0.0.0/0 ldap \
|
||||||
|
ldapserver=ldap.example.com \
|
||||||
|
ldapport=636 \
|
||||||
|
ldaptls=1 \
|
||||||
|
ldapbasedn="dc=example,dc=com" \
|
||||||
|
ldapbinddn="cn=neuraldb,dc=example,dc=com" \
|
||||||
|
ldapbindpasswd=bindpassword \
|
||||||
|
ldapsearchfilter="(sAMAccountName=%s)"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Auditing
|
||||||
|
|
||||||
|
Enable connection and statement auditing:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
CREATE EXTENSION pgaudit;
|
||||||
|
```
|
||||||
|
|
||||||
|
```ini
|
||||||
|
# neuraldb.conf
|
||||||
|
pgaudit.log = 'ddl, write, role'
|
||||||
|
pgaudit.log_catalog = on
|
||||||
|
pgaudit.log_client = on
|
||||||
|
pgaudit.log_level = log
|
||||||
|
```
|
||||||
|
|
||||||
|
Audit logs are written to the standard NeuralDB log file in a structured format, suitable for ingestion by SIEM systems.
|
||||||
185
neuraldb-docs/pages/config-replication.md
Normal file
185
neuraldb-docs/pages/config-replication.md
Normal file
|
|
@ -0,0 +1,185 @@
|
||||||
|
---
|
||||||
|
title: Replication
|
||||||
|
sort: 130
|
||||||
|
section-id: configuration
|
||||||
|
keywords: replication, primary, replica, streaming replication, multi-region, consistency
|
||||||
|
description: Configuring NeuralDB replication — primary/replica setup, multi-region, and consistency levels
|
||||||
|
language: en
|
||||||
|
---
|
||||||
|
|
||||||
|
# Replication
|
||||||
|
|
||||||
|
NeuralDB replication is based on streaming replication: the primary continuously ships WAL records to replicas, which apply them in real time. This page explains how to set up and configure replication.
|
||||||
|
|
||||||
|
## Prerequisites
|
||||||
|
|
||||||
|
- The primary must have `wal_level = replica` or higher
|
||||||
|
- `max_wal_senders` must be greater than the number of replicas
|
||||||
|
- A replication user must exist
|
||||||
|
|
||||||
|
## Setting Up a Primary
|
||||||
|
|
||||||
|
Configure `neuraldb.conf` on the primary:
|
||||||
|
|
||||||
|
```ini
|
||||||
|
# neuraldb.conf (primary)
|
||||||
|
wal_level = replica
|
||||||
|
max_wal_senders = 10
|
||||||
|
max_replication_slots = 10
|
||||||
|
wal_keep_size = 1GB
|
||||||
|
hot_standby_feedback = on # prevents primary from vacuuming rows still needed by replicas
|
||||||
|
```
|
||||||
|
|
||||||
|
Create a replication user:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
CREATE USER replicator WITH REPLICATION PASSWORD 'repl-password';
|
||||||
|
```
|
||||||
|
|
||||||
|
Allow the replica to connect:
|
||||||
|
|
||||||
|
```
|
||||||
|
# pg_hba.conf (primary)
|
||||||
|
host replication replicator replica-ip/32 scram-sha-256
|
||||||
|
```
|
||||||
|
|
||||||
|
## Setting Up a Replica
|
||||||
|
|
||||||
|
On the replica server, use `pg_basebackup` to clone the primary:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# On the replica server
|
||||||
|
pg_basebackup \
|
||||||
|
--host=primary.example.com \
|
||||||
|
--port=5432 \
|
||||||
|
--username=replicator \
|
||||||
|
--pgdata=/var/lib/neuraldb/data \
|
||||||
|
--wal-method=stream \
|
||||||
|
--checkpoint=fast \
|
||||||
|
--progress \
|
||||||
|
--write-recovery-conf
|
||||||
|
```
|
||||||
|
|
||||||
|
The `--write-recovery-conf` flag creates a `standby.signal` file and writes connection info to `postgresql.auto.conf`, which tells NeuralDB to start in standby mode.
|
||||||
|
|
||||||
|
Configure `neuraldb.conf` on the replica:
|
||||||
|
|
||||||
|
```ini
|
||||||
|
# neuraldb.conf (replica)
|
||||||
|
hot_standby = on # allow read queries
|
||||||
|
hot_standby_feedback = on # send feedback to primary
|
||||||
|
wal_receiver_timeout = 60s
|
||||||
|
recovery_min_apply_delay = 0 # apply WAL immediately (increase for delayed replicas)
|
||||||
|
```
|
||||||
|
|
||||||
|
Start the replica:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
systemctl start neuraldb
|
||||||
|
```
|
||||||
|
|
||||||
|
Verify replication is working:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- On the primary
|
||||||
|
SELECT client_addr, state, sent_lsn, write_lsn, flush_lsn, replay_lsn,
|
||||||
|
(sent_lsn - replay_lsn) AS replication_lag_bytes
|
||||||
|
FROM pg_stat_replication;
|
||||||
|
```
|
||||||
|
|
||||||
|
## Replication Slots
|
||||||
|
|
||||||
|
Replication slots ensure the primary retains WAL until the replica has consumed it. This prevents the replica from falling too far behind, but also prevents WAL from being cleaned up if the replica disconnects.
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- Create a replication slot
|
||||||
|
SELECT pg_create_physical_replication_slot('replica_1');
|
||||||
|
|
||||||
|
-- List slots and their lag
|
||||||
|
SELECT slot_name, active, pg_size_pretty(pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn)) AS lag
|
||||||
|
FROM pg_replication_slots;
|
||||||
|
|
||||||
|
-- Drop a slot (do this if a replica is permanently removed)
|
||||||
|
SELECT pg_drop_replication_slot('replica_1');
|
||||||
|
```
|
||||||
|
|
||||||
|
**Warning:** Monitor slot lag. An inactive slot with large lag will cause unbounded WAL accumulation and can fill your disk.
|
||||||
|
|
||||||
|
## Synchronous Replication
|
||||||
|
|
||||||
|
By default, replication is asynchronous — the primary does not wait for replicas to acknowledge writes. For zero data loss, configure synchronous replication:
|
||||||
|
|
||||||
|
```ini
|
||||||
|
# neuraldb.conf (primary)
|
||||||
|
synchronous_standby_names = 'FIRST 1 (replica1, replica2)'
|
||||||
|
# ^ Wait for at least 1 of the listed standbys to acknowledge each commit
|
||||||
|
```
|
||||||
|
|
||||||
|
Modes:
|
||||||
|
- `FIRST n (list)` — wait for the first n standbys in the list
|
||||||
|
- `ANY n (list)` — wait for any n standbys from the list
|
||||||
|
- `*` — wait for all standbys
|
||||||
|
|
||||||
|
Per-transaction override:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SET synchronous_commit = 'local'; -- this transaction doesn't wait for replicas
|
||||||
|
```
|
||||||
|
|
||||||
|
## Multi-Region Replication
|
||||||
|
|
||||||
|
For global deployments, replicate to remote regions. The configuration is identical to local replication, but network latency affects synchronous commit performance.
|
||||||
|
|
||||||
|
Recommended approach for multi-region:
|
||||||
|
|
||||||
|
```
|
||||||
|
Primary (us-east-1)
|
||||||
|
├── Sync replica (us-east-1-az2) ← HA within region, ~2ms latency
|
||||||
|
├── Async replica (eu-west-1) ← EU reads, ~80ms latency
|
||||||
|
└── Async replica (ap-northeast-1) ← APAC reads, ~170ms latency
|
||||||
|
```
|
||||||
|
|
||||||
|
```ini
|
||||||
|
# Synchronous only within region; async to remote regions
|
||||||
|
synchronous_standby_names = 'FIRST 1 (local_replica)'
|
||||||
|
```
|
||||||
|
|
||||||
|
Configure the remote replicas with a `primary_conninfo` pointing to the primary:
|
||||||
|
|
||||||
|
```ini
|
||||||
|
# standby.signal (on replica)
|
||||||
|
primary_conninfo = 'host=primary.us-east-1.example.com port=5432 user=replicator password=repl-password sslmode=require'
|
||||||
|
```
|
||||||
|
|
||||||
|
## Failover
|
||||||
|
|
||||||
|
NeuralDB does not include automatic failover out of the box. Use one of:
|
||||||
|
|
||||||
|
- **Patroni** — industry-standard HA manager for PostgreSQL-compatible databases
|
||||||
|
- **NeuralDB HA Operator** — Kubernetes operator with automatic failover (see [Kubernetes docs](install-kubernetes.md))
|
||||||
|
- **repmgr** — lightweight failover manager
|
||||||
|
|
||||||
|
Manual failover:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# On the replica that will become the new primary
|
||||||
|
neuraldb-cli -c "SELECT pg_promote();"
|
||||||
|
```
|
||||||
|
|
||||||
|
After promotion, update `primary_conninfo` on all other replicas to point to the new primary.
|
||||||
|
|
||||||
|
## Monitoring Replication
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- Replication lag in bytes and seconds
|
||||||
|
SELECT client_addr, state,
|
||||||
|
pg_size_pretty(sent_lsn - replay_lsn) AS lag_bytes,
|
||||||
|
now() - pg_last_xact_replay_timestamp() AS lag_time
|
||||||
|
FROM pg_stat_replication;
|
||||||
|
|
||||||
|
-- On a replica: check its own lag
|
||||||
|
SELECT now() - pg_last_xact_replay_timestamp() AS lag,
|
||||||
|
pg_is_in_recovery() AS is_replica;
|
||||||
|
```
|
||||||
|
|
||||||
|
Set up an alert when replication lag exceeds 30 seconds.
|
||||||
228
neuraldb-docs/pages/config-server.md
Normal file
228
neuraldb-docs/pages/config-server.md
Normal file
|
|
@ -0,0 +1,228 @@
|
||||||
|
---
|
||||||
|
title: Server Config
|
||||||
|
sort: 100
|
||||||
|
section-id: configuration
|
||||||
|
keywords: neuraldb.conf, server configuration, settings, parameters, tuning
|
||||||
|
description: Complete reference for neuraldb.conf — all server configuration settings explained
|
||||||
|
language: en
|
||||||
|
---
|
||||||
|
|
||||||
|
# Server Config
|
||||||
|
|
||||||
|
NeuralDB is configured through `neuraldb.conf`, a key-value text file. This page documents all configuration parameters.
|
||||||
|
|
||||||
|
## Locating the Config File
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Show the active config file path
|
||||||
|
neuraldb-cli -c "SHOW config_file;"
|
||||||
|
```
|
||||||
|
|
||||||
|
Default locations:
|
||||||
|
- Linux: `/etc/neuraldb/neuraldb.conf`
|
||||||
|
- macOS Homebrew: `$(brew --prefix)/etc/neuraldb/neuraldb.conf`
|
||||||
|
- Docker: `/var/lib/neuraldb/data/neuraldb.conf`
|
||||||
|
|
||||||
|
Changes to `neuraldb.conf` require a reload (for most parameters) or a restart:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Reload without restart (applies most parameters)
|
||||||
|
neuraldb-cli -c "SELECT pg_reload_conf();"
|
||||||
|
|
||||||
|
# Full restart (required for listen_addresses, port, shared_buffers, etc.)
|
||||||
|
systemctl restart neuraldb
|
||||||
|
```
|
||||||
|
|
||||||
|
## Connection Settings
|
||||||
|
|
||||||
|
```ini
|
||||||
|
# Network interface to listen on
|
||||||
|
# '*' = all interfaces, 'localhost' = local only
|
||||||
|
listen_addresses = '*'
|
||||||
|
|
||||||
|
# TCP port
|
||||||
|
port = 5432
|
||||||
|
|
||||||
|
# Maximum simultaneous connections
|
||||||
|
# Each connection uses ~5 MB of memory
|
||||||
|
max_connections = 100
|
||||||
|
|
||||||
|
# Unix domain socket directory (Linux/macOS only)
|
||||||
|
unix_socket_directories = '/var/run/neuraldb'
|
||||||
|
|
||||||
|
# Superuser reserved connections
|
||||||
|
# Reserves this many connections exclusively for superusers
|
||||||
|
superuser_reserved_connections = 3
|
||||||
|
```
|
||||||
|
|
||||||
|
## Memory Settings
|
||||||
|
|
||||||
|
These are the most impactful parameters for performance.
|
||||||
|
|
||||||
|
```ini
|
||||||
|
# Page cache for relational data (row store)
|
||||||
|
# Recommended: 25% of available RAM
|
||||||
|
shared_buffers = 4GB
|
||||||
|
|
||||||
|
# Memory for HNSW vector indexes
|
||||||
|
# Recommended: 40-60% of available RAM for vector-heavy workloads
|
||||||
|
# Must be large enough to fit all active HNSW graphs
|
||||||
|
vector_buffer = 8GB
|
||||||
|
|
||||||
|
# Per-query working memory (sorts, hash joins)
|
||||||
|
# Recommended: 64MB–256MB for OLTP, more for analytical queries
|
||||||
|
# Be conservative: (max_connections × work_mem) should fit in RAM
|
||||||
|
work_mem = 128MB
|
||||||
|
|
||||||
|
# Memory for DDL maintenance (CREATE INDEX, VACUUM, etc.)
|
||||||
|
maintenance_work_mem = 2GB
|
||||||
|
|
||||||
|
# Shared memory for parallel query workers
|
||||||
|
parallel_query_mem = 512MB
|
||||||
|
```
|
||||||
|
|
||||||
|
## Vector Settings
|
||||||
|
|
||||||
|
```ini
|
||||||
|
# Default HNSW ef_search parameter (candidates evaluated per query)
|
||||||
|
# Higher = better recall, slower queries
|
||||||
|
hnsw.ef_search = 40
|
||||||
|
|
||||||
|
# Maximum number of vectors per shard before auto-splitting
|
||||||
|
vector_shard_size = 10000000
|
||||||
|
|
||||||
|
# Enable approximate nearest neighbour by default (true = HNSW index)
|
||||||
|
# Set to 'exact' to always use exact search (ignores vector indexes)
|
||||||
|
vector_scan = 'approximate'
|
||||||
|
|
||||||
|
# Number of parallel threads for HNSW index builds
|
||||||
|
hnsw.build_threads = 4
|
||||||
|
|
||||||
|
# Compression algorithm for stored vector data
|
||||||
|
# 'none', 'lz4', 'scalar_quantize' (lossy but 4× smaller)
|
||||||
|
vector_compression = 'lz4'
|
||||||
|
```
|
||||||
|
|
||||||
|
## WAL Settings
|
||||||
|
|
||||||
|
```ini
|
||||||
|
# WAL logging level
|
||||||
|
# 'minimal': minimum for crash recovery
|
||||||
|
# 'replica': enables streaming replication (default)
|
||||||
|
# 'logical': enables logical replication and CDC
|
||||||
|
wal_level = replica
|
||||||
|
|
||||||
|
# WAL segment size (change requires initdb)
|
||||||
|
wal_segment_size = 128MB
|
||||||
|
|
||||||
|
# Maximum number of concurrent WAL sender processes
|
||||||
|
max_wal_senders = 10
|
||||||
|
|
||||||
|
# Number of WAL segments to keep for standby catching up
|
||||||
|
wal_keep_size = 1GB
|
||||||
|
|
||||||
|
# Synchronise WAL to disk before acknowledging commit
|
||||||
|
# 'on': safest; 'local': only local disk; 'off': async (fastest, tiny durability risk)
|
||||||
|
synchronous_commit = on
|
||||||
|
|
||||||
|
# Interval between WAL checkpoints
|
||||||
|
checkpoint_timeout = 5min
|
||||||
|
checkpoint_completion_target = 0.9
|
||||||
|
max_wal_size = 4GB
|
||||||
|
```
|
||||||
|
|
||||||
|
## Replication Settings
|
||||||
|
|
||||||
|
```ini
|
||||||
|
# Comma-separated list of standby names that must acknowledge writes
|
||||||
|
# Leave empty for asynchronous replication
|
||||||
|
synchronous_standby_names = ''
|
||||||
|
|
||||||
|
# Maximum lag allowed before primary throttles writes
|
||||||
|
max_standby_lag = 30s
|
||||||
|
|
||||||
|
# Hot standby: allow reads on replicas
|
||||||
|
hot_standby = on
|
||||||
|
```
|
||||||
|
|
||||||
|
## Query Planner
|
||||||
|
|
||||||
|
```ini
|
||||||
|
# Estimated cost of a random page fetch (tune based on SSD vs HDD)
|
||||||
|
# Lower values favour index scans; higher values favour sequential scans
|
||||||
|
random_page_cost = 1.1 # NVMe SSD (default is 4.0 for HDD)
|
||||||
|
|
||||||
|
# Effective size of the disk cache (affects planner estimates)
|
||||||
|
effective_cache_size = 24GB # ~75% of RAM
|
||||||
|
|
||||||
|
# Enable parallel query
|
||||||
|
max_parallel_workers_per_gather = 4
|
||||||
|
max_parallel_workers = 8
|
||||||
|
max_worker_processes = 16
|
||||||
|
|
||||||
|
# Hybrid query planner behaviour
|
||||||
|
# 'auto': planner decides pre-filter vs post-filter
|
||||||
|
# 'pre-filter': always pre-filter
|
||||||
|
# 'post-filter': always post-filter
|
||||||
|
vector_hybrid_strategy = 'auto'
|
||||||
|
```
|
||||||
|
|
||||||
|
## Logging
|
||||||
|
|
||||||
|
```ini
|
||||||
|
# Log destination
|
||||||
|
log_destination = 'stderr' # 'stderr', 'csvlog', 'jsonlog', 'syslog'
|
||||||
|
|
||||||
|
# Minimum severity to log
|
||||||
|
# 'DEBUG5' (verbose) → 'INFO' → 'WARNING' → 'ERROR' → 'FATAL'
|
||||||
|
log_min_messages = WARNING
|
||||||
|
|
||||||
|
# Log all SQL statements with duration above this threshold (ms)
|
||||||
|
# -1 = disable; 0 = log everything; 250 = log slow queries only
|
||||||
|
log_min_duration_statement = 250
|
||||||
|
|
||||||
|
# Log query parameters
|
||||||
|
log_parameters = off
|
||||||
|
|
||||||
|
# Log connection events
|
||||||
|
log_connections = on
|
||||||
|
log_disconnections = on
|
||||||
|
|
||||||
|
# Log lock waits longer than this (ms)
|
||||||
|
deadlock_timeout = 1s
|
||||||
|
log_lock_waits = on
|
||||||
|
```
|
||||||
|
|
||||||
|
## Full Configuration Example
|
||||||
|
|
||||||
|
```ini
|
||||||
|
# neuraldb.conf — Production settings for a 32 vCPU / 128 GB server
|
||||||
|
|
||||||
|
listen_addresses = '*'
|
||||||
|
port = 5432
|
||||||
|
max_connections = 500
|
||||||
|
superuser_reserved_connections = 5
|
||||||
|
|
||||||
|
shared_buffers = 32GB
|
||||||
|
vector_buffer = 64GB
|
||||||
|
work_mem = 128MB
|
||||||
|
maintenance_work_mem = 4GB
|
||||||
|
|
||||||
|
wal_level = replica
|
||||||
|
max_wal_senders = 10
|
||||||
|
wal_keep_size = 2GB
|
||||||
|
synchronous_commit = on
|
||||||
|
checkpoint_timeout = 10min
|
||||||
|
max_wal_size = 8GB
|
||||||
|
|
||||||
|
random_page_cost = 1.1
|
||||||
|
effective_cache_size = 96GB
|
||||||
|
max_parallel_workers_per_gather = 8
|
||||||
|
max_parallel_workers = 16
|
||||||
|
|
||||||
|
hnsw.ef_search = 80
|
||||||
|
vector_compression = lz4
|
||||||
|
|
||||||
|
log_min_duration_statement = 500
|
||||||
|
log_connections = on
|
||||||
|
```
|
||||||
221
neuraldb-docs/pages/config-storage.md
Normal file
221
neuraldb-docs/pages/config-storage.md
Normal file
|
|
@ -0,0 +1,221 @@
|
||||||
|
---
|
||||||
|
title: Storage Config
|
||||||
|
sort: 120
|
||||||
|
section-id: configuration
|
||||||
|
keywords: storage, memory, disk, SSD tiers, compression, tablespace, IOPS
|
||||||
|
description: Configuring NeuralDB storage — memory tiers, disk layout, compression, and tablespaces
|
||||||
|
language: en
|
||||||
|
---
|
||||||
|
|
||||||
|
# Storage Config
|
||||||
|
|
||||||
|
NeuralDB's performance depends heavily on storage configuration. This page covers how to optimise disk layout, configure memory tiers, enable compression, and use tablespaces for data tiering.
|
||||||
|
|
||||||
|
## Disk Layout Recommendations
|
||||||
|
|
||||||
|
Separate the data directory, WAL, and vector files onto different physical disks (or at least different volumes with guaranteed IOPS):
|
||||||
|
|
||||||
|
| Directory | Recommended storage | IOPS requirement |
|
||||||
|
|-----------|--------------------|--------------------||
|
||||||
|
| `$DATADIR/base/` | NVMe SSD | High random read/write |
|
||||||
|
| `$DATADIR/wal/` | NVMe SSD (separate) | High sequential write |
|
||||||
|
| `$DATADIR/vectors/` | NVMe SSD or RAM disk | High random read |
|
||||||
|
| `$DATADIR/archive/` | HDD or object storage | Low (sequential write) |
|
||||||
|
|
||||||
|
Configure separate paths in `neuraldb.conf`:
|
||||||
|
|
||||||
|
```ini
|
||||||
|
# Separate WAL onto a dedicated volume
|
||||||
|
wal_directory = '/mnt/nvme-wal/neuraldb/wal'
|
||||||
|
|
||||||
|
# Separate vector storage
|
||||||
|
vector_data_directory = '/mnt/nvme-fast/neuraldb/vectors'
|
||||||
|
|
||||||
|
# Archive destination for WAL shipping
|
||||||
|
archive_status_directory = '/var/lib/neuraldb/archive_status'
|
||||||
|
```
|
||||||
|
|
||||||
|
## Memory Tiers
|
||||||
|
|
||||||
|
NeuralDB has three distinct memory pools:
|
||||||
|
|
||||||
|
### shared_buffers (Row Store Cache)
|
||||||
|
|
||||||
|
The page cache for relational data. Sized at 25% of available RAM for a dedicated database server:
|
||||||
|
|
||||||
|
```ini
|
||||||
|
shared_buffers = 32GB # for a 128 GB server
|
||||||
|
```
|
||||||
|
|
||||||
|
### vector_buffer (Vector Index Cache)
|
||||||
|
|
||||||
|
Holds the HNSW graph in memory. The entire active HNSW graph must fit in `vector_buffer` for optimal performance. When the graph doesn't fit, NeuralDB falls back to disk-based graph traversal, which is 10–50× slower.
|
||||||
|
|
||||||
|
Calculate the required size:
|
||||||
|
|
||||||
|
```
|
||||||
|
vector_buffer = num_vectors × dimensions × 4 bytes × hnsw_overhead_factor
|
||||||
|
```
|
||||||
|
|
||||||
|
Where `hnsw_overhead_factor` ≈ 1.3 for default HNSW parameters (m=16).
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- Check current vector index memory usage
|
||||||
|
SELECT table_name, index_name,
|
||||||
|
pg_size_pretty(hnsw_graph_size_bytes) AS graph_memory,
|
||||||
|
pg_size_pretty(vector_data_size_bytes) AS data_size
|
||||||
|
FROM neuraldb_stat_vector_indexes;
|
||||||
|
```
|
||||||
|
|
||||||
|
### work_mem (Per-Query Buffer)
|
||||||
|
|
||||||
|
Used for in-memory sorts, hash joins, and bitmap operations. Set conservatively — each query can allocate multiple `work_mem` buffers:
|
||||||
|
|
||||||
|
```ini
|
||||||
|
# For 200 connections with typical 2 buffers per query:
|
||||||
|
# Max memory consumption: 200 × 2 × 128MB = 51 GB
|
||||||
|
work_mem = 128MB
|
||||||
|
```
|
||||||
|
|
||||||
|
Override per-session for analytical queries:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SET work_mem = '2GB';
|
||||||
|
SELECT ... complex analytical query ...
|
||||||
|
```
|
||||||
|
|
||||||
|
## Compression
|
||||||
|
|
||||||
|
### Row Store Compression
|
||||||
|
|
||||||
|
NeuralDB compresses SSTables on disk. Choose the algorithm based on your priorities:
|
||||||
|
|
||||||
|
```ini
|
||||||
|
# Compression algorithm for row data
|
||||||
|
# 'none': no compression (fastest reads/writes, most disk)
|
||||||
|
# 'lz4': fast, moderate compression ratio (~2-3×) — default
|
||||||
|
# 'zstd': slower compression, better ratio (~3-5×)
|
||||||
|
# 'zstd-9': high compression for archival (slow, ~6-8×)
|
||||||
|
storage_compression = lz4
|
||||||
|
|
||||||
|
# Compression level (for zstd only), 1-19
|
||||||
|
storage_compression_level = 3
|
||||||
|
```
|
||||||
|
|
||||||
|
### Vector Compression
|
||||||
|
|
||||||
|
```ini
|
||||||
|
# Vector data compression
|
||||||
|
# 'none': full precision, largest storage
|
||||||
|
# 'lz4': fast, minimal precision loss
|
||||||
|
# 'scalar_quantize': reduce to 8-bit (4× smaller, ~1% recall loss)
|
||||||
|
# 'product_quantize': very high compression, higher recall loss
|
||||||
|
vector_compression = lz4
|
||||||
|
```
|
||||||
|
|
||||||
|
To enable scalar quantisation for a specific index:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
CREATE INDEX ON documents
|
||||||
|
USING hnsw (embedding vector_cosine_ops)
|
||||||
|
WITH (quantization = 'scalar');
|
||||||
|
```
|
||||||
|
|
||||||
|
Scalar-quantised indexes use 4× less memory and may be faster due to better cache utilisation, at a typical recall cost of 0.5–2%.
|
||||||
|
|
||||||
|
## Tablespaces
|
||||||
|
|
||||||
|
Use tablespaces to store different tables or indexes on different volumes:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- Create tablespaces pointing to different mount points
|
||||||
|
CREATE TABLESPACE fast_ssd LOCATION '/mnt/nvme-fast';
|
||||||
|
CREATE TABLESPACE bulk_hdd LOCATION '/mnt/hdd-storage';
|
||||||
|
|
||||||
|
-- Create a table on fast SSD
|
||||||
|
CREATE TABLE hot_documents (
|
||||||
|
id UUID PRIMARY KEY,
|
||||||
|
content TEXT,
|
||||||
|
embedding VECTOR(1536)
|
||||||
|
) TABLESPACE fast_ssd;
|
||||||
|
|
||||||
|
-- Move an index to a specific tablespace
|
||||||
|
CREATE INDEX ON hot_documents USING hnsw (embedding vector_cosine_ops)
|
||||||
|
TABLESPACE fast_ssd;
|
||||||
|
|
||||||
|
-- Move old partitions to cheaper storage
|
||||||
|
ALTER TABLE documents_2024_q1 SET TABLESPACE bulk_hdd;
|
||||||
|
```
|
||||||
|
|
||||||
|
## Table Partitioning
|
||||||
|
|
||||||
|
Partition large tables by time or tenant to improve query performance and manageability:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- Partition documents by month
|
||||||
|
CREATE TABLE documents (
|
||||||
|
id UUID NOT NULL DEFAULT gen_random_uuid(),
|
||||||
|
content TEXT,
|
||||||
|
embedding VECTOR(1536),
|
||||||
|
tenant_id UUID,
|
||||||
|
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()
|
||||||
|
) PARTITION BY RANGE (created_at);
|
||||||
|
|
||||||
|
CREATE TABLE documents_2026_01
|
||||||
|
PARTITION OF documents
|
||||||
|
FOR VALUES FROM ('2026-01-01') TO ('2026-02-01')
|
||||||
|
TABLESPACE fast_ssd;
|
||||||
|
|
||||||
|
CREATE TABLE documents_2025_archive
|
||||||
|
PARTITION OF documents
|
||||||
|
FOR VALUES FROM ('2025-01-01') TO ('2026-01-01')
|
||||||
|
TABLESPACE bulk_hdd;
|
||||||
|
```
|
||||||
|
|
||||||
|
## VACUUM and Autovacuum
|
||||||
|
|
||||||
|
NeuralDB inherits PostgreSQL's MVCC-based VACUUM system:
|
||||||
|
|
||||||
|
```ini
|
||||||
|
# Autovacuum settings
|
||||||
|
autovacuum = on
|
||||||
|
autovacuum_naptime = 1min
|
||||||
|
autovacuum_vacuum_threshold = 50
|
||||||
|
autovacuum_vacuum_scale_factor = 0.05 # vacuum when 5% of rows are dead
|
||||||
|
autovacuum_analyze_threshold = 50
|
||||||
|
autovacuum_analyze_scale_factor = 0.02 # analyse when 2% of rows change
|
||||||
|
|
||||||
|
# For high-write tables, increase autovacuum aggressiveness
|
||||||
|
autovacuum_vacuum_cost_delay = 2ms # default: 20ms
|
||||||
|
autovacuum_vacuum_cost_limit = 400 # default: 200
|
||||||
|
```
|
||||||
|
|
||||||
|
Manually vacuum a table:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
VACUUM ANALYZE documents; -- reclaim space + update statistics
|
||||||
|
VACUUM FULL documents; -- full rewrite (blocking, very thorough)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Monitoring Disk Usage
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- Table sizes
|
||||||
|
SELECT table_name,
|
||||||
|
pg_size_pretty(pg_total_relation_size(quote_ident(table_name))) AS total,
|
||||||
|
pg_size_pretty(pg_relation_size(quote_ident(table_name))) AS table,
|
||||||
|
pg_size_pretty(pg_total_relation_size(quote_ident(table_name))
|
||||||
|
- pg_relation_size(quote_ident(table_name))) AS indexes
|
||||||
|
FROM information_schema.tables
|
||||||
|
WHERE table_schema = 'public'
|
||||||
|
ORDER BY pg_total_relation_size(quote_ident(table_name)) DESC;
|
||||||
|
|
||||||
|
-- Vector index sizes
|
||||||
|
SELECT * FROM neuraldb_stat_vector_indexes
|
||||||
|
ORDER BY hnsw_graph_size_bytes DESC;
|
||||||
|
|
||||||
|
-- Bloat estimate
|
||||||
|
SELECT tablename, n_dead_tup, last_vacuum, last_autovacuum
|
||||||
|
FROM pg_stat_user_tables
|
||||||
|
ORDER BY n_dead_tup DESC;
|
||||||
|
```
|
||||||
125
neuraldb-docs/pages/index.md
Normal file
125
neuraldb-docs/pages/index.md
Normal file
|
|
@ -0,0 +1,125 @@
|
||||||
|
---
|
||||||
|
title: What is NeuralDB?
|
||||||
|
sort: 100
|
||||||
|
section-id: overview
|
||||||
|
keywords: NeuralDB, introduction, AI database, vector database, overview
|
||||||
|
description: What NeuralDB is, its key use cases, and a high-level architecture overview
|
||||||
|
language: en
|
||||||
|
---
|
||||||
|
|
||||||
|
# What is NeuralDB?
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
NeuralDB is an AI-native database that unifies vector storage, semantic search, and relational data management in a single, horizontally scalable system. It is designed for applications that need to combine structured data queries with the semantic understanding that modern AI models provide.
|
||||||
|
|
||||||
|
Traditional databases store and retrieve data based on exact matches and range queries. NeuralDB adds a third dimension: **semantic proximity**. You can ask "find the 20 products most similar to this description" at the same time as "where inventory > 0 and price < 100" — in a single, atomic query with full ACID guarantees.
|
||||||
|
|
||||||
|
## Why NeuralDB Exists
|
||||||
|
|
||||||
|
The AI application stack of 2024–2026 exposed a fundamental tension in data architecture. Teams building retrieval-augmented generation (RAG) systems, recommendation engines, and semantic search needed:
|
||||||
|
|
||||||
|
- A **vector database** (Pinecone, Weaviate, Qdrant) for embedding storage and similarity search
|
||||||
|
- A **relational database** (PostgreSQL, MySQL) for structured metadata and transactions
|
||||||
|
- A **synchronisation layer** to keep the two in sync — a constant source of bugs and operational overhead
|
||||||
|
|
||||||
|
NeuralDB replaces all three with a single system. Vectors and relational data share the same storage engine, the same transaction log, and the same query planner. There is no synchronisation problem because there is no synchronisation needed.
|
||||||
|
|
||||||
|
## Key Capabilities
|
||||||
|
|
||||||
|
### Hybrid Vector + Relational Queries
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT id, name, price, SIMILARITY(embedding, :query_embedding) AS score
|
||||||
|
FROM products
|
||||||
|
WHERE category = 'electronics'
|
||||||
|
AND price BETWEEN 50 AND 500
|
||||||
|
AND stock > 0
|
||||||
|
ORDER BY score DESC
|
||||||
|
LIMIT 10;
|
||||||
|
```
|
||||||
|
|
||||||
|
This query uses a vector index for semantic ranking and a B-tree index for the relational filters, with the query planner choosing the optimal execution order.
|
||||||
|
|
||||||
|
### Multiple Vector Index Types
|
||||||
|
|
||||||
|
NeuralDB supports three similarity metrics out of the box:
|
||||||
|
|
||||||
|
| Metric | Use case |
|
||||||
|
|--------|----------|
|
||||||
|
| Cosine similarity | Text embeddings, normalised vectors |
|
||||||
|
| Dot product | Recommendation systems (unnormalised) |
|
||||||
|
| Euclidean (L2) | Image embeddings, spatial data |
|
||||||
|
|
||||||
|
### Full ACID Transactions
|
||||||
|
|
||||||
|
Unlike most vector databases, NeuralDB provides full ACID guarantees — including transactional upserts of both the relational row and the vector embedding simultaneously.
|
||||||
|
|
||||||
|
### Automatic Embedding Updates
|
||||||
|
|
||||||
|
Configure NeuralDB to call an embedding API whenever a text column changes:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
ALTER TABLE documents
|
||||||
|
ADD COLUMN embedding VECTOR(1536)
|
||||||
|
GENERATED ALWAYS AS EMBEDDING OF content
|
||||||
|
USING openai_ada_002;
|
||||||
|
```
|
||||||
|
|
||||||
|
NeuralDB handles the embedding pipeline automatically — no application-level embedding code required.
|
||||||
|
|
||||||
|
### Multi-Modal Vectors
|
||||||
|
|
||||||
|
Store and query vectors from any modality — text, image, audio, or custom — in the same table:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
CREATE TABLE media_items (
|
||||||
|
id UUID PRIMARY KEY,
|
||||||
|
title TEXT,
|
||||||
|
text_embedding VECTOR(1536),
|
||||||
|
image_embedding VECTOR(512),
|
||||||
|
created_at TIMESTAMP DEFAULT NOW()
|
||||||
|
);
|
||||||
|
```
|
||||||
|
|
||||||
|
## Use Cases
|
||||||
|
|
||||||
|
**Retrieval-Augmented Generation (RAG)** — Store your document corpus with embeddings. Query the most relevant chunks in a single round-trip and inject them into your LLM prompt.
|
||||||
|
|
||||||
|
**Semantic Product Search** — Replace keyword search with semantic search. Find products matching "comfortable running shoes for flat feet" even if those exact words don't appear in any product description.
|
||||||
|
|
||||||
|
**Recommendation Engines** — Store user preference vectors alongside item vectors. Compute collaborative-filtering recommendations with a single NQL query.
|
||||||
|
|
||||||
|
**Anomaly Detection** — Flag records whose vectors are distant from the cluster of "normal" data.
|
||||||
|
|
||||||
|
**Duplicate Detection** — Find near-duplicate records across millions of rows using approximate nearest-neighbour (ANN) search.
|
||||||
|
|
||||||
|
**Knowledge Graphs** — Store entity embeddings alongside relationship metadata for graph-enhanced retrieval.
|
||||||
|
|
||||||
|
## NeuralDB vs Traditional Approaches
|
||||||
|
|
||||||
|
| Capability | NeuralDB | Postgres + pgvector | Pinecone + Postgres |
|
||||||
|
|-----------|---------|---------------------|---------------------|
|
||||||
|
| Vector search | Native, HNSW | Extension, limited | Native |
|
||||||
|
| Relational queries | Full SQL | Full SQL | None (separate DB) |
|
||||||
|
| Hybrid queries | Single query | Single query | Application-layer join |
|
||||||
|
| ACID transactions | Yes | Yes | Partial |
|
||||||
|
| Horizontal sharding | Built-in | Manual (Citus) | Managed |
|
||||||
|
| Automatic embeddings | Yes | No | No |
|
||||||
|
| Streaming ingestion | Yes | No | Partial |
|
||||||
|
|
||||||
|
## Getting Started
|
||||||
|
|
||||||
|
The fastest way to try NeuralDB is with Docker:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker run -p 5432:5432 -e NEURALDB_PASSWORD=mypassword neuraldb/neuraldb:latest
|
||||||
|
```
|
||||||
|
|
||||||
|
Connect with any PostgreSQL-compatible client:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
psql -h localhost -U neuraldb -d neuraldb
|
||||||
|
```
|
||||||
|
|
||||||
|
Then read the [Core Concepts](concepts.md) to understand the NeuralDB data model, or jump to [NQL Basics](nql-basics.md) to start writing queries.
|
||||||
66
neuraldb-docs/theme.yml
Normal file
66
neuraldb-docs/theme.yml
Normal file
|
|
@ -0,0 +1,66 @@
|
||||||
|
# mdcms theme — default
|
||||||
|
# Edit colours, fonts, and layout here. See docs for full reference.
|
||||||
|
|
||||||
|
# ──────────────────────────────────
|
||||||
|
# Colours
|
||||||
|
# ──────────────────────────────────
|
||||||
|
light:
|
||||||
|
accent: "#2563EB"
|
||||||
|
background: "#FFFFFF"
|
||||||
|
nav-background: "#F8FAFC"
|
||||||
|
text: "#1E293B"
|
||||||
|
text-muted: "#64748B"
|
||||||
|
|
||||||
|
dark:
|
||||||
|
accent: "#60A5FA"
|
||||||
|
background: "#0F172A"
|
||||||
|
nav-background: "#1E293B"
|
||||||
|
text: "#F1F5F9"
|
||||||
|
text-muted: "#94A3B8"
|
||||||
|
|
||||||
|
# ──────────────────────────────────
|
||||||
|
# Semantic colours
|
||||||
|
# Used by callout tags (info, warning, success, error).
|
||||||
|
# Choose values that work on both light and dark backgrounds.
|
||||||
|
# ──────────────────────────────────
|
||||||
|
colours-semantic:
|
||||||
|
info: "#2563EB"
|
||||||
|
warning: "#D97706"
|
||||||
|
success: "#16A34A"
|
||||||
|
error: "#DC2626"
|
||||||
|
|
||||||
|
# ──────────────────────────────────
|
||||||
|
# Callout defaults
|
||||||
|
# ──────────────────────────────────
|
||||||
|
callouts:
|
||||||
|
info:
|
||||||
|
icon: info
|
||||||
|
primary-colour: "#2563EB"
|
||||||
|
background-colour: "#2563EB"
|
||||||
|
warning:
|
||||||
|
icon: warning
|
||||||
|
primary-colour: "#D97706"
|
||||||
|
background-colour: "#D97706"
|
||||||
|
success:
|
||||||
|
icon: success
|
||||||
|
primary-colour: "#16A34A"
|
||||||
|
background-colour: "#16A34A"
|
||||||
|
error:
|
||||||
|
icon: error
|
||||||
|
primary-colour: "#DC2626"
|
||||||
|
background-colour: "#DC2626"
|
||||||
|
|
||||||
|
# ──────────────────────────────────
|
||||||
|
# Typography
|
||||||
|
# Format: "provider:Font Name:weight" (provider: bunny | google)
|
||||||
|
# ──────────────────────────────────
|
||||||
|
font-body: "bunny:Noto Sans:400"
|
||||||
|
font-heading: "bunny:Noto Sans:700"
|
||||||
|
font-size: 1.0 # unitless multiplier (1.0 = 16px base)
|
||||||
|
line-height: 1.7 # unitless multiplier
|
||||||
|
|
||||||
|
# ──────────────────────────────────
|
||||||
|
# Layout
|
||||||
|
# ──────────────────────────────────
|
||||||
|
main-width: 80em
|
||||||
|
nav-width: 20em
|
||||||
Loading…
Reference in a new issue