diff --git a/latest/linux/mdcms b/latest/linux/mdcms index f8f11b2..f93aeae 100644 Binary files a/latest/linux/mdcms and b/latest/linux/mdcms differ diff --git a/latest/linux/mdcms.deb b/latest/linux/mdcms.deb index 3cfa335..cc568e8 100644 Binary files a/latest/linux/mdcms.deb and b/latest/linux/mdcms.deb differ diff --git a/latest/macos/mdcms b/latest/macos/mdcms index f4f5ecc..8ea55eb 100644 Binary files a/latest/macos/mdcms and b/latest/macos/mdcms differ diff --git a/latest/windows/mdcms.exe b/latest/windows/mdcms.exe index a8757ed..06706e4 100644 Binary files a/latest/windows/mdcms.exe and b/latest/windows/mdcms.exe differ diff --git a/sample-sites/README.md b/sample-sites/README.md new file mode 100644 index 0000000..12139f2 --- /dev/null +++ b/sample-sites/README.md @@ -0,0 +1,12 @@ +## Add folder info using this document + +* The contents of a **Readme.md** will show up embedded on the top of the folder it is in (in the web interface and the mobile apps) +* Formatting is supported with the bar on top (using Markdown) +* It uses Nextcloud Text so you can collaborate on it ๐ŸŽ‰ +* You can use and remix the templates as you like, they are in the public domain via the [CC0 license](https://creativecommons.org/publicdomain/zero/1.0/) + +## Action items + +* [ ] Try out the new templates +* [ ] Add your own templates in this folder +* [ ] โ€ฆ diff --git a/sample-sites/kitchen-table/assets/images/bread.jpg b/sample-sites/kitchen-table/assets/images/bread.jpg new file mode 100644 index 0000000..52b268c Binary files /dev/null and b/sample-sites/kitchen-table/assets/images/bread.jpg differ diff --git a/sample-sites/kitchen-table/assets/images/hero.jpg b/sample-sites/kitchen-table/assets/images/hero.jpg new file mode 100644 index 0000000..fa020c4 Binary files /dev/null and b/sample-sites/kitchen-table/assets/images/hero.jpg differ diff --git a/sample-sites/kitchen-table/assets/images/market.jpg b/sample-sites/kitchen-table/assets/images/market.jpg new file mode 100644 index 0000000..39b1c3b Binary files /dev/null and b/sample-sites/kitchen-table/assets/images/market.jpg differ diff --git a/sample-sites/kitchen-table/assets/images/pasta.jpg b/sample-sites/kitchen-table/assets/images/pasta.jpg new file mode 100644 index 0000000..9c2fc9e Binary files /dev/null and b/sample-sites/kitchen-table/assets/images/pasta.jpg differ diff --git a/sample-sites/kitchen-table/config.yml b/sample-sites/kitchen-table/config.yml new file mode 100644 index 0000000..f914c73 --- /dev/null +++ b/sample-sites/kitchen-table/config.yml @@ -0,0 +1,6 @@ +# mdcms v0.3 | DO NOT REMOVE THIS COMMENT +sitename: The Kitchen Table +sitedescription: Recipes, techniques, and stories from Amelia Fontaine +navigation: topbar +search: true +footer: "ยฉ 2026 Amelia Fontaine ยท The Kitchen Table" diff --git a/samplesite/index.html b/sample-sites/kitchen-table/index.html similarity index 76% rename from samplesite/index.html rename to sample-sites/kitchen-table/index.html index c2b1e8a..1466873 100644 --- a/samplesite/index.html +++ b/sample-sites/kitchen-table/index.html @@ -1,5 +1,6 @@ + + + + + + + +MD-CMS + + + + + + + + + + + + + + +
+ + + + + + diff --git a/sample-sites/modern-philosophy/nav.yml b/sample-sites/modern-philosophy/nav.yml new file mode 100644 index 0000000..8004459 --- /dev/null +++ b/sample-sites/modern-philosophy/nav.yml @@ -0,0 +1,204 @@ +# nav.yml โ€” generated by mdcms.py + +sections: + - code: front-matter + defaultname: Front Matter + sort: 50 + pagesvisibility: visible + + - code: epistemology + defaultname: Part I โ€” Epistemology + sort: 100 + pagesvisibility: visible + + - code: metaphysics + defaultname: Part II โ€” Metaphysics + sort: 200 + pagesvisibility: visible + + - code: ethics + defaultname: Part III โ€” Ethics + sort: 300 + pagesvisibility: visible + + - code: conclusion + defaultname: Conclusion + sort: 400 + pagesvisibility: visible + +pages: + - file: pages/preface.md + title: Preface + section-id: front-matter + sort: 100 + variants: [en] + titles: + en: Preface + + - file: pages/how-to-use.md + title: How to Use This Book + section-id: front-matter + sort: 110 + variants: [en] + titles: + en: How to Use This Book + + - file: pages/ep-01-knowledge.md + title: What is Knowledge? + section-id: epistemology + sort: 100 + variants: [en] + titles: + en: What is Knowledge? + + - file: pages/ep-02-perception.md + title: Perception and Reality + section-id: epistemology + sort: 110 + variants: [en] + titles: + en: Perception and Reality + + - file: pages/ep-03-reason.md + title: Reason and Rationalism + section-id: epistemology + sort: 120 + variants: [en] + titles: + en: Reason and Rationalism + + - file: pages/ep-04-empiricism.md + title: Empiricism + section-id: epistemology + sort: 130 + variants: [en] + titles: + en: Empiricism + + - file: pages/ep-05-scepticism.md + title: Scepticism and Its Responses + section-id: epistemology + sort: 140 + variants: [en] + titles: + en: Scepticism and Its Responses + + - file: pages/ep-06-truth.md + title: Theories of Truth + section-id: epistemology + sort: 150 + variants: [en] + titles: + en: Theories of Truth + + - file: pages/meta-01-existence.md + title: Existence and Being + section-id: metaphysics + sort: 100 + variants: [en] + titles: + en: Existence and Being + + - file: pages/meta-02-identity.md + title: Identity and Persistence + section-id: metaphysics + sort: 110 + variants: [en] + titles: + en: Identity and Persistence + + - file: pages/meta-03-causation.md + title: Causation + section-id: metaphysics + sort: 120 + variants: [en] + titles: + en: Causation + + - file: pages/meta-04-freewill.md + title: Free Will and Determinism + section-id: metaphysics + sort: 130 + variants: [en] + titles: + en: Free Will and Determinism + + - file: pages/meta-05-mind.md + title: Philosophy of Mind + section-id: metaphysics + sort: 140 + variants: [en] + titles: + en: Philosophy of Mind + + - file: pages/meta-06-time.md + title: The Nature of Time + section-id: metaphysics + sort: 150 + variants: [en] + titles: + en: The Nature of Time + + - file: pages/eth-01-foundations.md + title: Foundations of Ethics + section-id: ethics + sort: 100 + variants: [en] + titles: + en: Foundations of Ethics + + - file: pages/eth-02-consequentialism.md + title: Consequentialism + section-id: ethics + sort: 110 + variants: [en] + titles: + en: Consequentialism + + - file: pages/eth-03-deontology.md + title: Deontological Ethics + section-id: ethics + sort: 120 + variants: [en] + titles: + en: Deontological Ethics + + - file: pages/eth-04-virtue.md + title: Virtue Ethics + section-id: ethics + sort: 130 + variants: [en] + titles: + en: Virtue Ethics + + - file: pages/eth-05-applied.md + title: Applied Ethics + section-id: ethics + sort: 140 + variants: [en] + titles: + en: Applied Ethics + + - file: pages/eth-06-political.md + title: Political Philosophy + section-id: ethics + sort: 150 + variants: [en] + titles: + en: Political Philosophy + + - file: pages/synthesis.md + title: Synthesis and Open Questions + section-id: conclusion + sort: 100 + variants: [en] + titles: + en: Synthesis and Open Questions + + - file: pages/further-reading.md + title: Further Reading + section-id: conclusion + sort: 110 + variants: [en] + titles: + en: Further Reading diff --git a/sample-sites/modern-philosophy/pages/ep-01-knowledge.md b/sample-sites/modern-philosophy/pages/ep-01-knowledge.md new file mode 100644 index 0000000..0587f1c --- /dev/null +++ b/sample-sites/modern-philosophy/pages/ep-01-knowledge.md @@ -0,0 +1,55 @@ +--- +title: "What is Knowledge?" +sort: 100 +section-id: epistemology +description: The JTB analysis of knowledge, the Gettier problem, and the major responses to Gettier. +language: en +--- + +# What is Knowledge? + +Epistemology โ€” from the Greek *episteme* (knowledge) and *logos* (account or study) โ€” is the branch of philosophy concerned with the nature, sources, and limits of knowledge. Its central question is deceptively simple: what is it to know something? + +## The Traditional Analysis: Justified True Belief + +The dominant account in Western philosophy from Plato through most of the twentieth century held that knowledge is *justified true belief* (hereafter JTB). To know that *p* is to (1) believe that *p*, (2) have *p* be true, and (3) be justified in believing that *p*. + +Each condition plays a role. The truth condition rules out lucky coincidences: if I believe the train departs at 10am and it in fact departs at 10am, I do not know this if I formed the belief by guessing. The belief condition rules out propositions I accept without endorsing: I may act as if London is south of Edinburgh (it is not) without believing this, in which case I cannot be said to know it. The justification condition โ€” the most philosophically contested of the three โ€” distinguishes knowledge from mere true belief: if I believe, on no grounds whatsoever, that there is a spider behind the bookcase, and there happens to be a spider there, I do not thereby *know* there is a spider. Knowledge requires that one's belief be appropriately supported. + +The JTB analysis has Platonic roots: in the *Meno*, Socrates distinguishes knowledge (*episteme*) from right opinion (*ortho doxa*) by the presence of an "account" that tethers the belief to its object. In the *Theaetetus*, Plato examines and ultimately rejects several definitions of knowledge, leaving the question famously open. + +## The Gettier Problem + +In a short, devastating paper published in 1963, Edmund Gettier showed that the JTB analysis is insufficient ^[Gettier, E., "Is Justified True Belief Knowledge?", *Analysis* 23, 1963, pp.121-123]. He produced two counterexamples โ€” cases where an agent has a justified true belief but, intuitively, does not know. + +The original cases are somewhat technical, but their structure can be illustrated as follows. Smith has good evidence that Jones will get the job, and that Jones has ten coins in his pocket. He infers: "The man who will get the job has ten coins in his pocket." In fact, Smith himself gets the job โ€” and unbeknownst to him, he also has ten coins in his pocket. Smith's belief is true and justified. But he does not know it, because his justification for the belief is the evidence about Jones, not about himself. The truth of his belief is, in the relevant sense, a matter of luck. + +Gettier cases share a common structure: the belief is true, and it is justified, but the justification and the truth are *accidentally* connected in a way that undermines knowledge. The epistemic luck that disqualifies knowledge is sometimes called *veritic luck* โ€” the belief could easily have been false, even given the justification. + +## Responses to Gettier + +The philosophical literature on Gettier is vast ^[For surveys, see Shope, R., *The Analysis of Knowing*, Princeton UP, 1983; Ichikawa, J. and Steup, M., "The Analysis of Knowledge", *Stanford Encyclopedia of Philosophy*, 2018]. Several broad strategies have been pursued. + +**The No-False-Lemmas Approach.** Some responses add a fourth condition: knowledge requires that the justification not pass through any false beliefs. In the Smith-Jones case, Smith's inference passes through the false belief that Jones will get the job. This approach handles many Gettier cases but fails against variants that generate knowledge through no false belief. + +**Defeasibility Theories.** Lehrer and Paxson proposed that knowledge requires that one's justification not be *defeatable* by true information ^[Lehrer, K. and Paxson, T., "Knowledge: Undefeated Justified True Belief", *Journal of Philosophy* 66, 1969, pp.225-237]. If there is some truth that, were the agent to learn it, would undermine her justification, she does not know. This captures the intuition that Gettier cases involve misleading justification, but the correct formulation of the defeasibility condition has proven elusive. + +**Reliabilism.** Alvin Goldman proposed replacing the traditional internalist justification condition with an externalist one: knowledge requires that the belief be produced by a *reliable belief-forming process* ^[Goldman, A., "What is Justified Belief?", in *Justification and Knowledge*, ed. Pappas, Reidel, 1979]. A reliable process is one that tends to produce true beliefs in the actual world. Perception, memory, and valid inference are typically reliable; guessing and wishful thinking are not. Reliabilism handles Gettier cases naturally: if a belief is produced by a reliable process, there is no epistemic luck. + +**Safety and Sensitivity Conditions.** Sosa and Nozick proposed modal conditions on knowledge. Nozick's *tracking theory* required that the belief "tracks" the truth: if *p* were false, the agent would not believe *p* (the sensitivity condition); and if *p* were true, the agent would believe *p* (the adherence condition) ^[Nozick, R., *Philosophical Explanations*, Harvard UP, 1981, ch.3]. Sosa's *safety* condition required that the agent could not easily have been wrong ^[Sosa, E., "How to Defeat Opposition to Moore", *Philosophical Perspectives* 13, 1999]. + +**Knowledge First.** Timothy Williamson has argued that the traditional project of analysing knowledge in terms of more basic conditions is fundamentally misguided ^[Williamson, T., *Knowledge and Its Limits*, Oxford UP, 2000]. Knowledge, he argues, is a primitive mental state โ€” not reducible to belief plus conditions. Rather than asking what conditions must supplement belief to yield knowledge, we should take knowledge as the starting point and explain belief and justification in terms of it. The slogan is "knowledge first." + +## Internalism and Externalism + +The Gettier debate brought into focus a broader dispute about the nature of epistemic justification. *Internalists* hold that the factors that determine whether a belief is justified must be *accessible* to the agent โ€” available through reflection alone ^[Chisholm, R., *Theory of Knowledge*, Prentice-Hall, 1966]. On this view, two agents who are internally identical (same beliefs, same phenomenal states) must be equally justified, even if their environments differ dramatically. + +*Externalists* deny this. Goldman's reliabilism is paradigmatically externalist: whether a belief is produced by a reliable process is a fact about the external world, not something the agent can determine by reflection. An agent might have a perfectly reliable belief-forming process that she has no way of knowing is reliable. + +The internalism/externalism debate intersects with questions about scepticism (discussed in Chapter 5). Externalism offers a natural reply to sceptical scenarios โ€” brain-in-a-vat believers may have reliably formed beliefs even in their abnormal environment โ€” but faces the challenge of explaining the felt force of sceptical intuitions, which seem to appeal precisely to considerations accessible by reflection. + +## Knowledge and Understanding + +A growing body of work distinguishes *knowledge that* (propositional knowledge) from *knowledge how* (ability knowledge) and from *understanding*. Ryle's distinction between knowing-that and knowing-how influentially challenged the assumption that all knowledge is propositional ^[Ryle, G., *The Concept of Mind*, Hutchinson, 1949]. Understanding โ€” grasping why something is the case, how the pieces fit together โ€” seems to go beyond a collection of propositional beliefs and is increasingly seen as a distinct epistemic achievement worthy of investigation in its own right. + +The question "What is knowledge?" turns out, as Plato suspected, to be genuinely difficult. The Gettier problem demonstrated that the most natural answer โ€” justified true belief โ€” is insufficient, and the subsequent fifty years of philosophy have not produced a consensus on what must replace it. But the failure to find a reductive analysis does not mean we have learned nothing. We have learned precisely why the question is hard, and that is progress. diff --git a/sample-sites/modern-philosophy/pages/ep-02-perception.md b/sample-sites/modern-philosophy/pages/ep-02-perception.md new file mode 100644 index 0000000..e3db7d1 --- /dev/null +++ b/sample-sites/modern-philosophy/pages/ep-02-perception.md @@ -0,0 +1,61 @@ +--- +title: Perception and Reality +sort: 110 +section-id: epistemology +description: Direct realism, indirect realism, idealism, and phenomenalism โ€” the major theories of perception and its relation to reality. +language: en +--- + +# Perception and Reality + +Perception is our most immediate route to knowledge of the external world, and yet it is philosophically treacherous. We trust our senses โ€” and then we discover that sticks look bent in water, towers look small from a distance, and the table that appears brown under incandescent light appears subtly different under daylight. These illusions and variations prompt an epistemological crisis: if our senses can mislead us, how can we trust them? And if we cannot fully trust them, what can we know about the world? + +## The Argument from Illusion + +The *argument from illusion* is a traditional challenge to naive perceptual realism. It proceeds roughly as follows ^[Ayer, A.J., *The Foundations of Empirical Knowledge*, Macmillan, 1940]: + +1. In cases of illusion, what we are directly aware of (the bent stick, the shrunken tower) is not identical to the physical object. +2. Perceptual experience in veridical (non-illusory) cases is intrinsically similar to experience in illusory cases. +3. Therefore, what we are directly aware of even in veridical cases is not the physical object itself, but some intermediate object โ€” a *sense datum*, a subjective representation, a mental image. + +If this argument is correct, we never perceive the external world directly. We perceive representations of it, and must infer the world from those representations. + +## Direct Realism + +*Direct realism* (also called naรฏve realism or common-sense realism) holds that in ordinary perception, we are directly aware of the physical world. There are no intermediary mental objects standing between us and the things we perceive. + +Contemporary direct realists reject the argument from illusion by contesting its first premise. When I see the bent stick, I am not aware of some private sense datum; I am aware of the stick itself, and my experience has the representational content that the stick is bent โ€” which is a false content, but this does not require a separate object ^[Martin, M.G.F., "The Transparency of Experience", *Mind and Language* 17, 2002, pp.376-425]. + +**Disjunctivism** is a sophisticated variant of direct realism that draws a fundamental distinction between veridical experience and illusion/hallucination ^[McDowell, J., "Criteria, Defeasibility, and Knowledge", *Proceedings of the British Academy* 68, 1982]. On this view, there is no common factor between seeing a tree and hallucinating a tree. Veridical perception genuinely consists in being acquainted with the object; hallucination is a numerically distinct kind of event that merely mimics it. This dissolves the argument from illusion by denying that veridical and illusory experiences must have the same fundamental nature. + +## Indirect Realism + +*Indirect realism* (or representationalism) accepts that we never perceive the external world directly. Our direct objects of experience are mental representations โ€” sense data, *qualia*, or "ideas" in the empiricist terminology. These representations are caused by, and typically resemble, the physical objects that produce them. + +Locke is the canonical indirect realist in the early modern period ^[Locke, J., *An Essay Concerning Human Understanding*, 1689, Book II]. He distinguished *primary qualities* (extension, shape, motion, number) โ€” features of objects that genuinely resemble our ideas of them โ€” from *secondary qualities* (colour, taste, smell, temperature) โ€” features that our ideas do not resemble; they are simply the powers of objects to produce certain experiences in us. + +Indirect realism faces a significant epistemological challenge: if we only ever directly perceive our representations, how can we know that those representations accurately track the external world? Locke acknowledged this; Berkeley exploited it to devastating effect. + +## Berkeley's Idealism + +George Berkeley argued that indirect realism collapses into idealism ^[Berkeley, G., *A Treatise Concerning the Principles of Human Knowledge*, 1710]. If we only directly perceive ideas, and ideas are inherently mental, then matter โ€” that supposed cause of ideas existing independently of all minds โ€” is a philosopher's fiction. *Esse est percipi*: to be is to be perceived. + +Berkeley was not denying the existence of the ordinary objects of experience โ€” tables, trees, other people. He was claiming that their existence consists in their being perceived, either by finite minds or, when unobserved by us, by the mind of God. This is idealism, but of a commonsensical variety: Berkeley insisted his view was closer to common sense than Locke's. + +The main objection to Berkeley is the arbitrariness of experience. If physical objects are collections of ideas, why do we not simply experience whatever we imagine? Berkeley's answer โ€” the regularity of experience is guaranteed by God โ€” is metaphysically expensive and not universally persuasive. + +## Phenomenalism + +*Phenomenalism* is a non-theistic descendant of Berkeley, associated with Hume, Mill, and twentieth-century logical empiricists like A.J. Ayer. Rather than reducing physical objects to ideas in God's mind, phenomenalism analyses statements about physical objects as equivalent to conditionals about what experiences would occur under certain conditions ^[Mill, J.S., *An Examination of Sir William Hamilton's Philosophy*, 1865; Ayer, A.J., *Language, Truth and Logic*, Gollancz, 1936]. + +"There is a table in the next room" is analysed as something like: "If anyone were to look in the next room under normal conditions, they would have table-experiences." The table is, in Mill's phrase, a "permanent possibility of sensation." + +Phenomenalism faces serious difficulties with conditionals involving unfulfillable antecedents and with the enormous complexity required to capture ordinary physical-object claims in purely phenomenal terms. It has largely been abandoned as a research programme. + +## Contemporary Debates + +Current philosophy of perception engages with cognitive science and debates about the *format* of perceptual representation (is it propositional? imagistic? iconic?), the *reach* of perception (does it extend to abstract objects, high-level properties, or is it limited to low-level sensory features?), and the relationship between perception and belief ^[Siegel, S., *The Richness of the Senses*, Oxford UP, 2010]. + +The *enactivist* tradition, drawing on Merleau-Ponty's phenomenology, challenges representationalism from a different direction: perception, on this view, is not a matter of constructing internal representations but of active engagement with the environment ^[Noรซ, A., *Action in Perception*, MIT Press, 2004]. + +The debate between direct and indirect realism remains active and unresolved. What is clear is that perception โ€” however it ultimately works โ€” does not give us a transparent window onto the world; it gives us something whose relationship to the world requires careful philosophical examination. diff --git a/sample-sites/modern-philosophy/pages/ep-03-reason.md b/sample-sites/modern-philosophy/pages/ep-03-reason.md new file mode 100644 index 0000000..dfbd2fe --- /dev/null +++ b/sample-sites/modern-philosophy/pages/ep-03-reason.md @@ -0,0 +1,59 @@ +--- +title: Reason and Rationalism +sort: 120 +section-id: epistemology +description: Descartes, Leibniz, and Kant โ€” the rationalist tradition, a priori knowledge, and the role of reason in epistemology. +language: en +--- + +# Reason and Rationalism + +*Rationalism* is the view that reason โ€” independent of or prior to sensory experience โ€” is a significant source of knowledge. The paradigmatic rationalist claim is that some truths can be known *a priori*: known on the basis of reason alone, without appeal to experience. Mathematics and logic are the clearest cases. That 7 + 5 = 12, or that if all humans are mortal and Socrates is human then Socrates is mortal โ€” these seem knowable by pure thought, without conducting experiments or making observations. + +## The A Priori / A Posteriori Distinction + +The distinction between *a priori* and *a posteriori* knowledge โ€” between knowledge independent of experience and knowledge dependent on it โ€” was systematised by Kant, though it has roots in earlier philosophy ^[Kant, I., *Critique of Pure Reason*, 1781/1787, B1-B6]. + +*A priori* knowledge is justified independently of experience. It includes logical truths, mathematical truths, and perhaps certain conceptual truths (a bachelor is unmarried). Crucially, a priori knowledge is typically characterised by *necessity* and *universality*: a priori propositions are true in all possible worlds and admit of no exceptions. + +*A posteriori* (or *empirical*) knowledge is justified by experience. Contingent facts about the world โ€” there are seven continents, water is Hโ‚‚O, the temperature today is 22ยฐC โ€” are known a posteriori. Such propositions could have been otherwise, and we know them by observing the world. + +Kant also introduced the analytic/synthetic distinction. An *analytic* judgment is one where the predicate is contained in the concept of the subject ("All bachelors are unmarried"). A *synthetic* judgment adds something beyond the subject concept ("The cat is on the mat"). Rationalists typically claim there is a priori synthetic knowledge โ€” knowledge that is both independent of experience and genuinely informative about the world. Kant thought mathematics and the principles of pure science were synthetic a priori. + +## Descartes and the Method of Doubt + +Renรฉ Descartes is the foundational figure of early modern rationalism. His *Meditations on First Philosophy* (1641) begins with systematic doubt: he resolves to suspend belief in anything he can doubt, to find, if anything survives, a foundation for knowledge that is absolutely certain ^[Descartes, R., *Meditations on First Philosophy*, AT VII:17-18]. + +The senses can deceive. Dreams can be indistinguishable from waking life. And most radically: could there be an evil demon, infinitely powerful and infinitely cunning, whose sole purpose is to deceive him? Under this hypothesis, even the truths of mathematics might be false. + +From this radical doubt, Descartes extracts one certain truth: *cogito ergo sum* โ€” "I think, therefore I am." ^[Descartes, R., *Discourse on the Method*, AT VI:32]. Even if a demon deceives me, the deceiving requires that I exist as a thinking thing. The *cogito* survives the most radical doubt. + +From this single certainty, Descartes attempts to rebuild knowledge. He argues for the existence of a benevolent God who would not systematically deceive him, thereby reinstating trust in clear and distinct perception. The circularity of this reconstruction โ€” using clear and distinct perception to prove God's existence, then using God's existence to validate clear and distinct perception โ€” has been widely noted and is known as the *Cartesian circle* ^[Arnauld, A., *Fourth Objections*, in Descartes, *Meditations*, AT VII:214]. + +Despite these difficulties, Descartes' contribution is foundational: he established the *epistemological turn* โ€” the idea that a systematic theory of knowledge is the prerequisite for metaphysics and science. + +## Leibniz: Necessary Truths and Monads + +Gottfried Wilhelm Leibniz distinguished *truths of reason* (necessary truths, knowable a priori, the opposite of which is impossible) from *truths of fact* (contingent truths, known a posteriori, the opposite of which is conceivable) ^[Leibniz, G.W., *Monadology*, ยง33-34, 1714]. + +For Leibniz, the basic furniture of reality consists of *monads* โ€” immaterial, indivisible, mind-like substances. Each monad perceives (in a broad sense) every other monad, though with varying degrees of clarity. The apparent causal interaction between things is, in reality, a *pre-established harmony* installed by God: things do not genuinely cause each other but are programmed to correspond. + +Leibniz's principle of *sufficient reason* โ€” there must be a sufficient reason for everything being as it is rather than otherwise โ€” is a cornerstone of his system and has remained influential in metaphysics and the philosophy of science. + +## Kant's Copernican Revolution + +Immanuel Kant transformed the rationalism/empiricism debate with his *Critique of Pure Reason* (1781). He accepted from the rationalists that there is genuine a priori knowledge and from the empiricists that all knowledge *begins* with experience. His synthesis: experience is possible only because the mind structures it using a priori *forms* (space and time) and *categories* (substance, causation, necessity, and others). + +Kant called this the *Copernican revolution* in philosophy ^[Kant, I., *Critique of Pure Reason*, Bxvi]: just as Copernicus moved the sun to the centre, Kant moved the knowing subject. We do not passively receive an already-structured world; we actively structure the world we experience, using the forms of intuition and the categories of the understanding. + +This generates *transcendental idealism*: objects as we know them (*phenomena*) are partly constituted by our cognitive apparatus. Things as they are in themselves (*noumena*) โ€” beyond the conditions of our experience โ€” are unknowable. + +The great achievement of Kant's epistemology is explaining how synthetic a priori knowledge is possible: mathematical and scientific principles are synthetic a priori because they describe the structure that the mind imposes on experience, not features of mind-independent reality. The cost is that our knowledge is bounded by the limits of possible experience. + +## The Analytic Critique + +The logical empiricists of the early twentieth century (Carnap, Schlick, Ayer) challenged the very possibility of synthetic a priori knowledge ^[Ayer, A.J., *Language, Truth and Logic*, ch.4]. On their view, apparent a priori knowledge either reduces to analytic truths (true by definition) or is meaningless. Mathematics is analytic โ€” true by virtue of the meanings of mathematical terms. There are no synthetic a priori truths. + +Quine's "Two Dogmas of Empiricism" (1951) challenged even the analytic/synthetic distinction itself, arguing that no proposition is immune from revision in light of experience ^[Quine, W.V.O., "Two Dogmas of Empiricism", *Philosophical Review* 60, 1951]. This radical empiricism has been broadly influential but is not without critics ^[Grice, P. and Strawson, P., "In Defense of a Dogma", *Philosophical Review* 65, 1956]. + +The status of a priori knowledge remains one of epistemology's central contested questions. diff --git a/sample-sites/modern-philosophy/pages/ep-04-empiricism.md b/sample-sites/modern-philosophy/pages/ep-04-empiricism.md new file mode 100644 index 0000000..b855e2a --- /dev/null +++ b/sample-sites/modern-philosophy/pages/ep-04-empiricism.md @@ -0,0 +1,59 @@ +--- +title: Empiricism +sort: 130 +section-id: epistemology +description: Locke, Berkeley, and Hume โ€” the empiricist tradition and the limits of sensory knowledge. +language: en +--- + +# Empiricism + +*Empiricism* is the epistemological view that knowledge derives from, and must be grounded in, sensory experience. Where rationalism privileges reason, empiricism insists that the mind at birth is a *tabula rasa* โ€” a blank slate โ€” and that all our concepts and knowledge are acquired through experience. The great British empiricists of the seventeenth and eighteenth centuries โ€” John Locke, George Berkeley, and David Hume โ€” explored this view with increasing rigour and found, perhaps to their own surprise, that it leads to deeply uncomfortable places. + +## Locke's Empiricism + +John Locke set the agenda for British empiricism in his *Essay Concerning Human Understanding* (1689). His starting point is a polemic against innate ideas: there are no ideas or principles inscribed in the mind from birth, contrary to what Descartes and Leibniz held ^[Locke, J., *Essay*, I.ii]. + +All ideas originate in experience, which Locke divides into two kinds: *sensation* (the senses providing ideas of external objects) and *reflection* (the mind observing its own operations โ€” thinking, doubting, willing, perceiving). From simple ideas given in experience, the mind constructs *complex ideas* by combination, abstraction, and relation. + +Locke's primary/secondary quality distinction (discussed in the previous chapter) is central to his epistemology. We have genuine knowledge only of relations among our own ideas; the extent to which our ideas correspond to mind-independent reality is limited to the primary qualities. + +Locke's *Essay* is a monument of systematic empiricism, but it contains significant tensions. His account of substance โ€” the "I know not what" that underlies the qualities we perceive โ€” sits uneasily with his empiricism, since no experience corresponds to substance itself. + +## Hume's Fork and Bundle Theory + +David Hume pushed empiricism to its systematic conclusions with greater rigour and less embarrassment about where they led. In the *Treatise of Human Nature* (1739-40) and the *Enquiry Concerning Human Understanding* (1748), he developed what has been called the most powerful case in the history of philosophy for the limits of reason and knowledge. + +*Hume's fork* divides all genuine claims to knowledge into two classes ^[Hume, D., *Enquiry*, ยง4]: + +1. **Relations of ideas** โ€” propositions knowable a priori by reason alone, whose denials are contradictions (mathematics, logic, conceptual truths). These are certain but tell us nothing about the actual world. + +2. **Matters of fact** โ€” propositions about the world, known a posteriori through experience. Their denials are conceivable. They are contingent and can only be known through experience. + +Hume's criterion of empirical significance follows: any meaningful claim is either a relation of ideas or a matter of fact. Claims that fit neither category โ€” much of traditional metaphysics, theology, and rationalist philosophy โ€” are, famously, "nothing but sophistry and illusion" ^[Hume, D., *Enquiry*, ยง12.3]. + +On the self, Hume is radically deflationary. When he introspects, he finds no impression of a persistent, unified self โ€” only "a bundle or collection of different perceptions, which succeed each other with inconceivable rapidity" ^[Hume, D., *Treatise*, I.iv.6]. The self, on his view, is a fiction constructed from the successive flow of impressions and ideas. Personal identity is a matter of psychological continuity, not a metaphysical substance. + +## The Problem of Induction + +Hume's most influential contribution to epistemology is his analysis of inductive inference. We routinely infer, from past regularities, what will happen in the future: the sun has risen every morning, so it will rise tomorrow. All known emeralds have been green, so the next emerald will be green. What justifies these inferences? + +Not deductive reason: there is no logical contradiction in the sun's failing to rise. Not experience: to justify induction by appeal to the fact that induction has worked before is circular โ€” it assumes the very principle in question ^[Hume, D., *Treatise*, I.iii.6]. + +Hume's conclusion: our habit of inductive inference is a psychological necessity โ€” we cannot help forming expectations from regularities โ€” but it has no rational justification. This is the *problem of induction*, sometimes called "Hume's guillotine" for the way it cuts off a seemingly obvious route to empirical knowledge. + +The problem of induction has proved enormously productive. Karl Popper's falsificationism โ€” the view that science proceeds by bold conjecture and attempted refutation rather than inductive generalisation โ€” was an explicit response ^[Popper, K., *The Logic of Scientific Discovery*, 1934]. Nelson Goodman's "new riddle of induction" showed that the problem was deeper than Hume recognised: even if induction is sometimes reliable, we need a further principle to determine which regularities to project onto the future ^[Goodman, N., *Fact, Fiction and Forecast*, 1955]. + +## Hume on Causation + +Hume's analysis of causation is equally influential. We believe that causes necessitate their effects โ€” that fire *must* produce heat, that one billiard ball *must* move another when struck. But examining our impressions, Hume finds no impression of *necessary connection* between events ^[Hume, D., *Enquiry*, ยง7]. We observe constant conjunction โ€” event A is always followed by event B โ€” and we observe the spatial and temporal contiguity of cause and effect. But necessity itself is never observed. + +Hume's account: the idea of necessary connection is a projection of our own psychological tendency to expect B after A, given repeated experience of their conjunction. The "necessary connection" is in us, not in the world. + +This has generated enormous controversy. The *regularity theory* of causation โ€” in Hume's footsteps โ€” holds that causation just is constant conjunction (plus contiguity and temporal priority). Counterfactual theories, mechanistic theories, and probabilistic theories have all been proposed as improvements. + +## Empiricism's Legacy + +Empiricism as a systematic research programme continues in analytic philosophy. The logical empiricists (Carnap, Schlick, Neurath) developed a sophisticated version oriented toward the philosophy of science. Quine's naturalised epistemology โ€” the view that epistemology is continuous with empirical psychology โ€” is the most radical empiricist programme of the twentieth century ^[Quine, W.V.O., "Epistemology Naturalized", in *Ontological Relativity and Other Essays*, 1969]. + +The permanent contribution of the classical empiricists is methodological: the insistence that philosophical claims be answerable to experience, and the willingness to follow the logic of that insistence into uncomfortable territory. diff --git a/sample-sites/modern-philosophy/pages/ep-05-scepticism.md b/sample-sites/modern-philosophy/pages/ep-05-scepticism.md new file mode 100644 index 0000000..f6adb47 --- /dev/null +++ b/sample-sites/modern-philosophy/pages/ep-05-scepticism.md @@ -0,0 +1,53 @@ +--- +title: Scepticism and Its Responses +sort: 140 +section-id: epistemology +description: Cartesian scepticism, the brain-in-a-vat scenario, contextualism, and relevant alternatives theories. +language: en +--- + +# Scepticism and Its Responses + +Scepticism is the philosophical position that knowledge โ€” or at least, some significant domain of knowledge โ€” is impossible. It has been a central problem in epistemology from antiquity to the present, partly because it is remarkably difficult to refute and partly because attempting to refute it has driven some of philosophy's most creative work. + +## Ancient Scepticism + +The ancient Greek sceptics โ€” Pyrrho of Elis, and later the Academic sceptics including Arcesilaus and Carneades โ€” argued that for any claim, there is an equally strong case for its denial, leaving the rational response one of *epochรฉ*: suspension of judgment ^[Sextus Empiricus, *Outlines of Pyrrhonism*, I.8-12, c.200 CE]. Suspension of judgment, they argued, brings *ataraxia* โ€” tranquillity, freedom from the anxiety that dogmatic belief produces. + +Ancient scepticism was primarily a practical philosophy โ€” a way of living without commitment to metaphysical positions. Modern scepticism takes a different form: it is primarily an epistemological challenge, asking whether knowledge is possible given the limitations of our access to reality. + +## Cartesian Scepticism + +Descartes' sceptical scenarios, introduced in the *Meditations* as methodological tools, have become the canonical statements of modern epistemological scepticism. The *dreaming argument*: I cannot rule out that I am dreaming right now, and if I might be dreaming, I cannot be certain that anything I currently believe is true ^[Descartes, *Meditations*, AT VII:19]. + +The *evil demon hypothesis* is more radical: suppose there is an infinitely powerful deceiving demon who ensures that all my beliefs โ€” including the deliverances of reason and mathematics โ€” are false. I cannot disprove this. Therefore, I cannot be certain of anything. + +The sceptical strategy exploits what has been called *epistemic closure*: if I know that P entails Q, and I know P, then I know Q. Equivalently: if I don't know Q, and P entails Q, then I don't know P ^[Nozick, R., *Philosophical Explanations*, p.204]. Sceptics argue: if I knew that I have hands, I would know that I am not a brain in a vat; I do not know that I am not a brain in a vat; therefore, I do not know that I have hands. + +## The Brain-in-a-Vat Scenario + +Hilary Putnam updated Descartes' evil demon into the brain-in-a-vat scenario ^[Putnam, H., *Reason, Truth and History*, 1981, ch.1]. Suppose my brain has been removed from my body, placed in a vat, and is being fed electrical signals by a supercomputer that simulates a complete reality. All my experiences are exactly as they would be in normal embodied life. + +Putnam argued โ€” controversially โ€” that this scenario is incoherent. A brain in a vat lacks the causal connections to the external world that are necessary for its terms to refer to external objects. "Water," as a brain-in-a-vat thinks it, does not refer to Hโ‚‚O โ€” it refers, at most, to the computer simulation. So a brain-in-a-vat thinking "I am not a brain in a vat" is producing a true sentence โ€” because its words do not refer to the things that would make it false. This semantic argument against scepticism has been widely discussed and contested. + +## Responses to Scepticism + +**Moorean Responses.** G.E. Moore's response to scepticism was blunt: we know more certainly that we have hands than we know any philosophical premise used in the argument for scepticism ^[Moore, G.E., "Proof of an External World", *Proceedings of the British Academy* 25, 1939]. The *modus ponens* can be run in either direction: from the premises to the sceptical conclusion, or from the falsity of the sceptical conclusion to the falsity of one of the premises. Moore insisted the latter is more reasonable. + +Wittgenstein developed a related response: certain propositions โ€” "There are physical objects," "The world has existed for many years" โ€” function as *hinges* that cannot be doubted within any practice of inquiry, because doubting them would not be coherent inquiry but something else entirely ^[Wittgenstein, L., *On Certainty*, ยง341, 1951]. + +**Relevant Alternatives.** Fred Dretske proposed that knowledge requires ruling out only *relevant* alternatives โ€” possibilities that are live given one's actual situation ^[Dretske, F., "Epistemic Operators", *Journal of Philosophy* 67, 1970]. The possibility that I am a brain in a vat is not a relevant alternative in ordinary contexts; I do not need to rule it out to know that I have hands. Scepticism artificially expands the class of alternatives that must be eliminated. + +**Contextualism.** David Lewis and Stewart Cohen developed contextualist responses: the standards for knowledge vary with context ^[Lewis, D., "Elusive Knowledge", *Australasian Journal of Philosophy* 74, 1996; Cohen, S., "How to be a Fallibilist", *Philosophical Perspectives* 2, 1988]. In ordinary contexts, we correctly say we know many things. In sceptical philosophical discussions, where very high standards are in play, those knowledge attributions are false โ€” but this does not undermine ordinary attributions, which operate at a lower standard. Scepticism is a local phenomenon of artificially elevated epistemic standards. + +**Externalist Responses.** If knowledge requires reliably produced beliefs (as reliabilism holds), then the sceptical demon scenario involves beliefs that are not reliably produced and therefore do not constitute knowledge. But Descartes' scenario is just that โ€” a scenario where knowledge fails. This does not show that we actually lack knowledge in the real world. + +## Closure Denial + +Nozick's tracking theory denied epistemic closure, which blocks the sceptical argument at its source ^[Nozick, R., *Philosophical Explanations*, pp.204-211]. On his account, I know I have hands because if I didn't have hands I wouldn't believe I did (the sensitivity condition). But I do not know I am not a brain in a vat โ€” because if I were a brain in a vat, I would still believe I wasn't (the sensitivity condition fails). Yet the failure to know the second proposition does not undermine knowledge of the first, because the inference from "I have hands" to "I am not a brain in a vat" is not knowledge-preserving on Nozick's account. + +This is elegant, but the denial of closure is philosophically costly and has not won widespread acceptance. + +## The Significance of Scepticism + +Scepticism matters not primarily because it is a live hypothesis that reflective people adopt, but because engaging with it illuminates the structure of our knowledge and the character of epistemic justification. The sceptical challenge to close our eyes and demonstrate that we know anything about the external world has driven epistemologists to produce their most careful accounts of justification, reliability, and the conditions for knowledge. Scepticism is philosophy's sharpening stone. diff --git a/sample-sites/modern-philosophy/pages/ep-06-truth.md b/sample-sites/modern-philosophy/pages/ep-06-truth.md new file mode 100644 index 0000000..9cdf860 --- /dev/null +++ b/sample-sites/modern-philosophy/pages/ep-06-truth.md @@ -0,0 +1,59 @@ +--- +title: Theories of Truth +sort: 150 +section-id: epistemology +description: Correspondence, coherence, pragmatist, and deflationary theories of truth. +language: en +--- + +# Theories of Truth + +What is truth? The question seems either trivially easy or impossibly hard. Everyone knows that to say something true is to say how things are. And yet when we ask what this "correspondence" between language and world consists in, or whether there might be truths that do not correspond to any independent reality, we find ourselves in deep philosophical waters. + +## The Correspondence Theory + +The *correspondence theory of truth* is the classical answer: a proposition (or belief, or statement) is true if and only if it corresponds to the facts ^[Aristotle, *Metaphysics*, 1011b26-28]. To say that snow is white is to say something true, because it corresponds to the fact that snow is white. + +The appeal of this theory is its fidelity to common sense. When we assert something, we are attempting to describe how things are, and truth is the property of succeeding in that attempt. + +The challenge is making "correspondence" precise. Early twentieth-century philosophy developed the *picture theory of meaning* (Wittgenstein's *Tractatus*, Russell's logical atomism) on which propositions picture facts by sharing their logical structure ^[Wittgenstein, L., *Tractatus Logico-Philosophicus*, 1921, ยง2.15; Russell, B., "The Philosophy of Logical Atomism", *Monist* 28, 1918]. This was technically ambitious but ultimately abandoned. + +The central difficulty for correspondence theories is the nature of *facts*. Are facts mind-independent features of the world? If so, what exactly are they, and how do they differ from merely true propositions? The proliferation of suspect entities (negative facts, disjunctive facts, mathematical facts) has made many philosophers wary. + +## The Coherence Theory + +The *coherence theory* identifies truth with coherence โ€” membership in a system of beliefs that are mutually consistent, mutually supporting, and comprehensive ^[Bradley, F.H., *Essays on Truth and Reality*, 1914]. On this view, a belief is true not because it corresponds to some external fact but because it coheres with the overall system of beliefs. + +The coherence theory is associated with British idealism (Bradley, Bosanquet) and has been influential in anti-realist philosophy more generally. Its appeal lies in removing the mysterious "correspondence" relation: truth is an internal relation among beliefs, not a relation between thought and world. + +Critics raise two main objections. First, coherence is not sufficient for truth: many consistent, mutually supporting sets of beliefs are simply false. The elaborate beliefs of a deeply mistaken scientific tradition may be perfectly coherent. Second, coherence is not necessary: isolated beliefs can be true without being embedded in a rich coherent system. + +## Pragmatist Theories + +William James and John Dewey developed *pragmatist* theories of truth that identified truth with what "works" โ€” what is expedient to believe, what guides successful action ^[James, W., "Pragmatism's Conception of Truth", in *Pragmatism*, 1907; Dewey, J., *Logic: The Theory of Inquiry*, 1938]. + +James's formulation is the most quotable: "True ideas are those that we can assimilate, validate, corroborate and verify. False ideas are those that we cannot." + +Pragmatism has been persistently misread as claiming that truth is whatever we find convenient to believe, or that powerful people's beliefs are therefore true. More charitably, pragmatism is the claim that our concept of truth is tied to the role beliefs play in guiding inquiry and action: there is no coherent notion of truth that is entirely divorced from human practice. + +The strongest objection is that some truths are inaccessible to human inquiry โ€” truths about the distant past, truths about microscopic phenomena not yet observed. If truth is tied to what we could in principle verify, we seem to be claiming that there are no truths about such matters, which is implausible. + +Peirce's more sophisticated version identified truth with what ideal inquiry would converge on in the long run ^[Peirce, C.S., "How to Make Our Ideas Clear", *Popular Science Monthly* 12, 1878]. This avoids the problem of currently inaccessible truths but introduces a counter-factual notion of "ideal inquiry" that is difficult to cash out. + +## The Deflationary Theories + +*Deflationary theories* โ€” including Ramsey's *redundancy theory*, Quine's *disquotational theory*, and Horwich's *minimalism* โ€” hold that "is true" adds nothing to a proposition ^[Ramsey, F.P., "Facts and Propositions", *Proceedings of the Aristotelian Society* Supp. Vol. 7, 1927; Horwich, P., *Truth*, Blackwell, 1990]. + +To say "it is true that snow is white" is simply to say "snow is white." The predicate "is true" serves a logical function โ€” enabling us to endorse propositions without repeating them, to quantify over propositions ("everything she said is true") โ€” but there is no deep property of *truth* to be analysed. + +The T-schema captures the deflationist insight: for any sentence S, "'S' is true if and only if S." There is nothing more to truth than this biconditional schema. + +Deflationists must explain how the truth predicate can do its logical work without denoting a substantive property. Critics also note that scientific realism seems to require a robust notion of truth โ€” the success of science is best explained by the approximate truth of scientific theories โ€” and deflationists have struggled to accommodate this. + +## Truth and Language + +The relationship between truth and language raises further questions. Alfred Tarski's semantic theory of truth ^[Tarski, A., "The Concept of Truth in Formalized Languages", 1935] provided a technically rigorous account for formal languages: "Snow is white" is true in language L if and only if snow is white. For natural languages, which are semantically open, Tarski's approach encounters the *Liar paradox* ("This sentence is false") and related self-referential difficulties. + +Contemporary philosophy of language has explored truth-conditional semantics (meaning just is truth conditions), pluralism about truth (different truth predicates for different domains โ€” factual, moral, mathematical), and relativism (truth relative to standards or contexts). + +The debate about truth intersects with debates about realism and anti-realism, metaphysics, and the philosophy of language. It is one of the crossroads of philosophy โ€” a place where multiple independent routes converge. diff --git a/sample-sites/modern-philosophy/pages/eth-01-foundations.md b/sample-sites/modern-philosophy/pages/eth-01-foundations.md new file mode 100644 index 0000000..d6c7928 --- /dev/null +++ b/sample-sites/modern-philosophy/pages/eth-01-foundations.md @@ -0,0 +1,81 @@ +--- +title: Foundations of Ethics +sort: 100 +section-id: ethics +description: Metaethics versus normative ethics, the question of moral realism, and why ethical theory matters for practical reasoning. +language: en +--- + +# Foundations of Ethics + +Ethics is the branch of philosophy concerned with questions about value, obligation, and the good life. Before we can adjudicate between competing moral theories โ€” utilitarian, Kantian, Aristotelian โ€” we must first examine what kind of inquiry ethics is. This is the domain of *metaethics*. + +## The Metaethical Questions + +Metaethics asks: What is the nature of moral claims? When we say "Torturing innocents is wrong," are we: + +1. Stating an objective fact about the world? +2. Expressing a subjective attitude? +3. Issuing a kind of command or prescription? +4. Doing something altogether different? + +These are not merely academic quibbles. The answer constrains what normative ethics can hope to achieve. If there are no moral facts, then ethical argument collapses into persuasion. If there are moral facts but we cannot know them, then ethical confidence is always epistemically precarious. + +## Moral Realism + +Moral realists hold that there are objective moral facts โ€” facts that hold independently of what any individual or culture believes. G.E. Moore (1903) argued that "good" is a simple, indefinable, non-natural property that we perceive through a kind of moral intuition.^[Moore, G.E. (1903). *Principia Ethica*. Cambridge University Press.] + +Naturalistic moral realists, by contrast, identify moral properties with natural properties. Cornell realists such as Peter Railton and Richard Boyd argue that moral terms refer to natural facts about human flourishing, desire-satisfaction, or social coordination.^[Boyd, R. (1988). "How to be a Moral Realist." In Sayre-McCord (ed.), *Essays on Moral Realism*.] + +The **open question argument** (Moore) challenges naturalism: for any natural property N, it is always an open question whether something that is N is thereby good. If "good" just meant "maximises pleasure," then "Is pleasure-maximising action good?" would be a tautology โ€” but it is not. This suggests that moral properties are not identical to natural ones. + +## Anti-Realist Positions + +### Expressivism +A.J. Ayer's *Language, Truth and Logic* (1936) presented the classic emotivist thesis: moral statements are not truth-apt at all. "Stealing is wrong" means something like "Boo, stealing!" โ€” it expresses disapproval rather than describing a fact.^[Ayer, A.J. (1936). *Language, Truth and Logic*. Gollancz.] + +Simon Blackburn developed *quasi-realism* to address the main objection to expressivism: that moral statements appear in contexts (conditionals, embedded clauses) where purely expressive readings are implausible. "If stealing is wrong, then getting your brother to steal for you is also wrong" cannot mean "If boo stealing, then boo getting your brother to steal." + +### Error Theory +J.L. Mackie (1977) accepted that moral statements purport to state facts but argued they are systematically false. There are no objective moral properties in the world. We are all making a kind of category error when we assert moral claims.^[Mackie, J.L. (1977). *Ethics: Inventing Right and Wrong*. Penguin.] Mackie's *argument from queerness* claims that objective moral properties would be entities of a very strange kind โ€” utterly unlike anything in the natural world โ€” and our capacity to know them would require an equally strange epistemic faculty. + +### Constructivism +Kantian constructivists (Christine Korsgaard, John Rawls) occupy a middle position: moral truths are not mind-independent facts discovered by intuition, but neither are they merely expressions of attitude. They are constructed through procedures of rational reflection or agreement under idealised conditions. Moral facts are *the output* of a normative procedure, not independently existing objects.^[Rawls, J. (1980). "Kantian Constructivism in Moral Theory." *Journal of Philosophy*, 77(9).] + +## Normative Ethics: An Overview + +Normative ethics asks: what ought we to do, and why? Three traditions dominate: + +| Tradition | Central Question | Key Figure | +|---|---|---| +| Consequentialism | What outcomes should we produce? | John Stuart Mill | +| Deontology | What duties bind us regardless of outcome? | Immanuel Kant | +| Virtue Ethics | What kind of person should I be? | Aristotle | + +Each tradition is examined in subsequent chapters. Here we note that they often converge in practice while diverging in their theoretical foundations โ€” a useful starting heuristic. + +## Moral Epistemology + +How do we come to know moral truths (assuming there are any)? Candidates include: + +**Moral intuition** โ€” Direct, non-inferential moral knowledge. Strong intuitions (that gratuitous cruelty is wrong) are treated as data points that any adequate theory must accommodate. The method of *reflective equilibrium* (Rawls) involves moving back and forth between intuitions and principles until they cohere. + +**Moral perception** โ€” On some realist accounts, we literally perceive moral properties as we perceive colours (though with a different faculty). This view faces difficulty explaining inter-subjective disagreement. + +**Reason alone** โ€” Kantians hold that moral knowledge is a priori, derived from pure practical reason. We shall examine this in detail in the chapter on deontology. + +## The Fact-Value Distinction + +Hume's famous observation โ€” that we cannot derive an "ought" from an "is" โ€” remains one of the most contested claims in metaethics.^[Hume, D. (1740). *A Treatise of Human Nature*, III.i.1.] If no purely factual description of the world entails a moral conclusion, then moral premises are always smuggled into ethical arguments. Naturalists must either deny the is-ought gap or explain why the gap does not undermine their position. + +## Relativism and Universalism + +*Cultural moral relativism* โ€” the descriptive claim that moral codes vary across cultures โ€” is well-documented. *Moral relativism* โ€” the normative claim that what is right depends on cultural norms โ€” is a separate and far more contested thesis. It generates self-refutation problems: if morality is relative, then the moral principle "we should not impose our moral views on other cultures" is itself only relatively binding. + +Universalists hold that certain moral truths โ€” concerning dignity, suffering, basic rights โ€” apply to all humans in all contexts. The debate between particularism and universalism remains unresolved. + +## Further Reading + +- Parfit, D. (2011). *On What Matters*, Vols. Iโ€“II. Oxford University Press. +- Schroeder, M. (2010). *Noncognitivism in Ethics*. Routledge. +- Sayre-McCord, G. (ed.) (1988). *Essays on Moral Realism*. Cornell University Press. diff --git a/sample-sites/modern-philosophy/pages/eth-02-consequentialism.md b/sample-sites/modern-philosophy/pages/eth-02-consequentialism.md new file mode 100644 index 0000000..d5dc2cc --- /dev/null +++ b/sample-sites/modern-philosophy/pages/eth-02-consequentialism.md @@ -0,0 +1,67 @@ +--- +title: Consequentialism +sort: 110 +section-id: ethics +description: Utilitarian and consequentialist ethics from Bentham and Mill to Peter Singer and contemporary debates about act versus rule consequentialism. +language: en +--- + +# Consequentialism + +Consequentialism is the family of ethical theories holding that the moral quality of an action is entirely determined by its consequences. The right action is whichever action produces the best outcome. This apparently simple thesis generates a remarkably rich โ€” and contested โ€” philosophical programme. + +## Bentham's Utilitarianism + +Jeremy Bentham (1748โ€“1832) founded classical utilitarianism on the *principle of utility*: actions are right insofar as they promote happiness, and wrong insofar as they promote unhappiness. By "happiness," Bentham meant pleasure and the absence of pain.^[Bentham, J. (1789). *Introduction to the Principles of Morals and Legislation*. Payne.] + +Bentham proposed the *felicific calculus* โ€” a method for quantifying pleasure and pain along seven dimensions: intensity, duration, certainty, propinquity, fecundity, purity, and extent. The theory is rigorously impartialist: "each to count for one and none for more than one." The pleasure of a street cleaner counts exactly as much as that of an aristocrat. + +## Mill's Refinements + +John Stuart Mill (1806โ€“1873) accepted the utilitarian framework but argued that pleasures differ not only in quantity but in quality. *Higher pleasures* โ€” intellectual enjoyment, moral sentiment, aesthetic experience โ€” are intrinsically more valuable than lower, merely sensory pleasures.^[Mill, J.S. (1863). *Utilitarianism*. Parker, Son, and Bourn.] "It is better to be Socrates dissatisfied than a fool satisfied." + +Mill also attempted a consequentialist defence of rights and justice: rights protect interests so important that no ordinary gain in welfare could justify violating them. Whether this defence succeeds โ€” whether rights can be grounded in utility without collapsing into mere policy instruments โ€” remains debated. + +## Act and Rule Consequentialism + +**Act consequentialism** holds that each individual action should be evaluated by its consequences. The right act is the one that, among all available alternatives, produces the greatest aggregate welfare. + +**Rule consequentialism** holds that we should follow rules whose general adoption would produce the best consequences. We do not evaluate each act individually but ask: "What rule, if generally followed, would produce the best outcomes?" Rule consequentialism preserves more intuitive commitments about promise-keeping and justice: keeping a promise may not maximise utility on a particular occasion, but a rule requiring promise-keeping generally does. + +The objection to act consequentialism is that it seems to justify intuitively monstrous acts whenever the mathematics works out. If torturing an innocent person would prevent a slightly larger number of harms, act consequentialism apparently demands it. The *separateness of persons* objection (Rawls) argues that consequentialism fails to respect the distinction between persons, treating them merely as vessels for welfare rather than as individuals with their own claims. + +## Peter Singer and Preference Utilitarianism + +Peter Singer (b. 1946) defends a preference utilitarianism that extends moral consideration to all sentient beings capable of having preferences.^[Singer, P. (1979). *Practical Ethics*. Cambridge University Press.] The boundary of the moral community is not species membership but sentience. Singer's argument for animal liberation, global poverty obligations, and euthanasia all follow from applying the impartial preference calculus rigorously. + +Singer's *drowning child* argument: if you could save a drowning child at trivial cost to yourself, you are morally required to do so. But the same logic applies to distant strangers dying of preventable diseases. If distance does not diminish moral obligation, affluent people in wealthy nations are obligated to give dramatically more than they typically do.^[Singer, P. (1972). "Famine, Affluence, and Morality." *Philosophy & Public Affairs*, 1(3).] + +## Objections + +### The Demandingness Objection +Impartial consequentialism seems to demand that we sacrifice almost all personal projects, relationships, and pleasures to maximise aggregate welfare. Bernard Williams argued that this alienates us from our own "ground projects" โ€” the commitments that give our lives meaning.^[Williams, B. (1973). "A Critique of Utilitarianism." In Smart & Williams, *Utilitarianism: For and Against*.] + +### The Integrity Objection +Williams' related argument: if consequences are all that matter, then I should be willing to perform any act โ€” including acts I find deeply repugnant โ€” if doing so maximises welfare. This seems to demand that agents violate their own integrity in ways that undermine the coherence of a moral life. + +### The Measurement Problem +How do we compare welfare across persons? Cardinal welfare comparisons are notoriously difficult. Preference satisfaction is a proxy, but preferences can be adaptive (the oppressed learn to desire less), malformed, or satisfied in ways that harm the agent. + +### Rights Violations +Robert Nozick's side-constraints view: there are moral side-constraints on action โ€” rights โ€” that cannot be overridden even by sufficiently large welfare gains. Using a person merely as a means to aggregate welfare violates their dignity as an end in themselves.^[Nozick, R. (1974). *Anarchy, State, and Utopia*. Basic Books.] + +## Sophisticated Consequentialism + +Many contemporary consequentialists have developed more sophisticated positions that accommodate common moral intuitions: + +- **Indirect consequentialism**: evaluate character traits and dispositions by their consequences, not individual acts +- **Two-level utilitarianism** (Hare): intuitive level rules for everyday decision-making, critical level for theoretical reflection +- **Satisficing consequentialism**: require producing good-enough outcomes rather than maximising + +These refinements preserve the spirit of consequentialism while avoiding the most counterintuitive implications. + +## Further Reading + +- Parfit, D. (1984). *Reasons and Persons*. Oxford University Press. +- Crisp, R. (1997). *Mill on Utilitarianism*. Routledge. +- Kagan, S. (1989). *The Limits of Morality*. Oxford University Press. diff --git a/sample-sites/modern-philosophy/pages/eth-03-deontology.md b/sample-sites/modern-philosophy/pages/eth-03-deontology.md new file mode 100644 index 0000000..bd79029 --- /dev/null +++ b/sample-sites/modern-philosophy/pages/eth-03-deontology.md @@ -0,0 +1,82 @@ +--- +title: Deontological Ethics +sort: 120 +section-id: ethics +description: Kant's categorical imperative, the formulas of universal law and humanity, perfect and imperfect duties, and neo-Kantian developments. +language: en +--- + +# Deontological Ethics + +Deontological ethics holds that certain actions are intrinsically right or wrong regardless of their consequences. The term derives from the Greek *deon* (duty). Immanuel Kant (1724โ€“1804) constructed the most influential deontological system in the history of ethics, grounding morality entirely in reason rather than sentiment or consequences. + +## Kant's Moral Philosophy: Starting Points + +Kant begins the *Groundwork of the Metaphysics of Morals* (1785) by identifying the only thing that is good without qualification: a **good will**.^[Kant, I. (1785). *Groundwork of the Metaphysics of Morals*. Trans. Korsgaard, Cambridge UP, 1998.] Intelligence, courage, and even happiness can be used for evil purposes. But a will that acts from duty โ€” that acts because doing so is right, regardless of inclination or consequence โ€” is good unconditionally. + +This distinguishes acting *in accordance with* duty (which a prudent merchant might do for self-interested reasons) from acting *from* duty (the only source of genuine moral worth). + +## The Categorical Imperative + +Kant argues that all genuine moral requirements are *categorical* imperatives โ€” commands that apply unconditionally, regardless of one's desires. "Pay your debts" is categorical: it applies whether or not you want to, whether or not it benefits you. By contrast, "If you want to be trusted, pay your debts" is a *hypothetical* imperative, binding only if you have the relevant desire. + +Kant offers three principal formulations of the categorical imperative, claiming they are equivalent: + +### Formula of Universal Law (FUL) +> "Act only according to that maxim whereby you can at the same time will that it should become a universal law." + +To test whether an action is permissible, extract the maxim (underlying principle) of the action and ask: could I consistently will that everyone act on this maxim? The classic example is lying promises. My maxim: "When in financial difficulty, I will make a false promise to repay a loan." If universalised, the institution of promising collapses โ€” no one would believe promises. The maxim is self-defeating when universalised. + +### Formula of Humanity (FH) +> "Act so that you treat humanity, whether in your own person or in that of another, always as an end and never as a means only." + +Persons have *dignity* โ€” a value beyond all price. Using someone merely as an instrument for your purposes violates their status as a rational, self-legislating agent. This formula generates more intuitive verdicts than FUL in many cases and grounds a robust conception of human rights. + +### Formula of the Kingdom of Ends (FKE) +> "Act according to maxims of a universally legislating member of a merely possible kingdom of ends." + +The moral community is a hypothetical kingdom of rational agents who legislate universal laws for themselves and for all. Each person is both subject to and author of the moral law. + +## Perfect and Imperfect Duties + +Kant distinguishes **perfect duties** (negative, admitting no exceptions) from **imperfect duties** (positive, allowing latitude in how they are fulfilled). + +- *Perfect duties*: Do not murder. Do not lie. Do not make false promises. These admit no exceptions. +- *Imperfect duties*: Develop your talents. Help others in need. We must pursue these ends, but have discretion in how. + +## The Formula of Universal Law: Applications + +Kant tests four cases: + +1. **Suicide to escape suffering** โ€” The maxim of self-destruction from self-love contradicts itself when universalised (life-preserving instinct cannot simultaneously mandate destroying life). +2. **False promises** โ€” Universalised, this destroys the institution of promising. +3. **Neglecting one's talents** โ€” Although we can consistently will a world where all neglect their talents, we cannot *rationally* will such a world as members who might need others' developed capacities. +4. **Refusing to aid others** โ€” We cannot rationally will a world with no mutual aid, since we might need it ourselves. + +## Objections to Kantian Ethics + +### The Problem of Conflicting Duties +What if telling the truth would lead to murder? The notorious example: a murderer asks you where your friend is hiding. Kant's strict application of FUL seems to require telling the truth.^[Kant, I. (1797). "On a Supposed Right to Lie from Philanthropy."] Most critics find this conclusion intolerable. Defenders argue Kant was wrong to apply his own theory in this case. + +### The Formalism Objection (Hegel) +Hegel objected that the categorical imperative is empty โ€” too formal to generate determinate moral content. Almost any maxim can be made consistent with FUL through reformulation.^[Hegel, G.W.F. (1821). *Philosophy of Right*, ยง135.] + +### The Rigorism Objection +The absolute prohibition on lying, even to prevent serious harm, seems morally obtuse. A moral theory that ignores consequences entirely cannot be adequate. + +### The Humanity Formula and Its Scope +Does FH extend to animals? Kant seems to deny that animals have dignity (since they lack rationality), but this generates counterintuitive implications about the permissibility of animal cruelty. + +## Neo-Kantian Developments + +**Christine Korsgaard** grounds Kantian ethics in the structure of reflective self-consciousness. When we act, we implicitly endorse a principle. Practical identity โ€” the source of all our obligations โ€” commits us to valuing humanity as an end.^[Korsgaard, C. (1996). *Sources of Normativity*. Cambridge University Press.] + +**Thomas Scanlon's contractualism**: An act is wrong if its performance under the circumstances would be disallowed by any set of principles that no one could reasonably reject.^[Scanlon, T.M. (1998). *What We Owe to Each Other*. Harvard University Press.] This grounds moral requirements in what we owe to each other as persons โ€” a broadly Kantian spirit without the metaphysical apparatus. + +**W.D. Ross** introduced the concept of *prima facie* duties โ€” duties that are binding unless overridden by stronger competing duties in a given situation. Fidelity, gratitude, non-maleficence, beneficence, and justice are among them. This pluralist deontology avoids the single-minded rigour of Kant while preserving the idea that some actions have moral weight independent of consequences.^[Ross, W.D. (1930). *The Right and the Good*. Oxford University Press.] + +## Further Reading + +- Korsgaard, C. (1996). *Creating the Kingdom of Ends*. Cambridge University Press. +- O'Neill, O. (1989). *Constructions of Reason*. Cambridge University Press. +- Herman, B. (1993). *The Practice of Moral Judgment*. Harvard University Press. diff --git a/sample-sites/modern-philosophy/pages/eth-04-virtue.md b/sample-sites/modern-philosophy/pages/eth-04-virtue.md new file mode 100644 index 0000000..d94a448 --- /dev/null +++ b/sample-sites/modern-philosophy/pages/eth-04-virtue.md @@ -0,0 +1,82 @@ +--- +title: Virtue Ethics +sort: 130 +section-id: ethics +description: Aristotle's eudaimonia, the virtues, the doctrine of the mean, and contemporary neo-Aristotelian revival in moral philosophy. +language: en +--- + +# Virtue Ethics + +Virtue ethics shifts the primary question of moral theory from "What should I do?" to "What kind of person should I be?" Rather than specifying rules or calculating consequences, virtue ethics focuses on the character traits โ€” the *virtues* โ€” that constitute human excellence and the good life. + +## Aristotle's Ethics + +The foundational text is Aristotle's *Nicomachean Ethics* (ca. 350 BCE).^[Aristotle. *Nicomachean Ethics*. Trans. Irwin, Hackett, 1999.] Aristotle begins with the observation that every action aims at some good. The highest good โ€” the good for its own sake โ€” he calls *eudaimonia*, usually translated as "happiness" but better rendered as *flourishing* or *living well*. + +Eudaimonia is not a feeling but an *activity*: the activity of the soul in accordance with virtue (*arete*). It is not a momentary state but characterises a complete life. + +## The Function Argument + +Aristotle argues that just as a knife has a function (cutting) and a good knife fulfils its function excellently, human beings have a characteristic function. The human function, unique among animals, is rational activity. *Eudaimonia* is the excellent exercise of our rational capacities.^[*NE* I.7, 1097b24โ€“1098a20.] + +This *ergon* (function) argument has been criticised for assuming that humans have a single, discoverable function. Nonetheless, it provides the teleological framework within which the virtues are defined. + +## The Doctrine of the Mean + +Virtues are stable dispositions of character that enable us to respond appropriately to situations. They are acquired through habituation: we become courageous by practising courageous acts. The virtuous person does not merely act rightly but does so with pleasure and without painful struggle โ€” virtue has been fully internalised. + +Each virtue is a **mean** (*mesotes*) between two extremes โ€” excess and deficiency. Courage is the mean between cowardice (deficiency of boldness) and rashness (excess). Generosity lies between miserliness and prodigality. The mean is not arithmetically fixed but *relative to us* โ€” what counts as appropriate depends on context and the individual. + +| Excess | Virtue | Deficiency | +|---|---|---| +| Rashness | Courage | Cowardice | +| Prodigality | Generosity | Miserliness | +| Vanity | Magnanimity | Pusillanimity | +| Obsequiousness | Friendliness | Quarrelsomeness | +| Buffoonery | Wit | Boorishness | + +## Practical Wisdom: Phronesis + +The master virtue in Aristotle's scheme is *phronesis* โ€” practical wisdom. The person of practical wisdom perceives what a situation requires, deliberates well about how to act, and acts accordingly. Practical wisdom is not reducible to following rules; it requires experience, perception, and judgment. + +This distinguishes virtue ethics from rule-based approaches: no finite set of rules can capture what the practically wise person knows. The virtuous person's perceptions and responses are *constitutive* of right action, not mere applications of antecedent principles. + +## The Unity of the Virtues + +Aristotle holds that the virtues are unified: one cannot genuinely have any virtue without practical wisdom, and practical wisdom requires all the virtues. This *unity thesis* is controversial โ€” it seems possible to be courageous but unjust. Defenders argue that only with the full integration of virtues does one have the "complete" versions; partial virtues are mere natural tendencies, not fully-fledged character traits. + +## The Neo-Aristotelian Revival + +Virtue ethics experienced a significant revival in twentieth-century analytic philosophy, partly as a reaction to the perceived limitations of both consequentialism and deontology. + +**G.E.M. Anscombe** (1958) argued that concepts like "moral obligation" and "duty" are residues of a divine-law framework that has been abandoned; without God, they are incoherent. We should return to Aristotelian concepts of virtue, human nature, and flourishing.^[Anscombe, G.E.M. (1958). "Modern Moral Philosophy." *Philosophy*, 33(124).] + +**Philippa Foot** developed a naturalistic virtue ethics grounding virtues in what is good for humans as the kind of organisms we are. *Natural goodness* is a matter of the proper functioning of organisms of a particular natural kind.^[Foot, P. (2001). *Natural Goodness*. Oxford University Press.] + +**Alasdair MacIntyre** (*After Virtue*, 1981) argued that contemporary moral discourse is fragmented and incoherent because we have lost the teleological framework within which virtue concepts made sense. The Enlightenment project of grounding morality in individual reason or sentiment was doomed to fail. We need to recover Aristotelian tradition โ€” structured around practices, narrative, and community โ€” to make ethics intelligible.^[MacIntyre, A. (1981). *After Virtue*. University of Notre Dame Press.] + +## Contemporary Virtue Ethics + +**Rosalind Hursthouse** has developed a virtue-theoretic account of right action: an action is right if it is what a virtuous person would characteristically do in the circumstances.^[Hursthouse, R. (1999). *On Virtue Ethics*. Oxford University Press.] This need not be circular: virtuous persons are those with character traits that constitute human flourishing, and we can characterise flourishing independently. + +**Michael Slote** defends agent-based virtue ethics, grounding moral evaluation entirely in the motivational states of agents rather than objective human flourishing. + +**Julia Annas** argues that virtue ethics is best understood not as a rival to rule-following but as an account of how we internalise moral requirements through the development of character. + +## Objections to Virtue Ethics + +### Action Guidance +Critics allege that virtue ethics provides insufficient guidance: when I face a difficult choice, "act as a virtuous person would" tells me little unless I already know what virtue requires. The response is that moral life is not primarily about making hard decisions but about forming character โ€” and character provides guidance of a richer kind than any algorithm. + +### Cultural Relativism +If virtues are defined relative to a human *telos* and that *telos* varies across cultures, different cultures will have different, potentially incompatible, lists of virtues. MacIntyre acknowledges this but argues that tradition-internal reasoning can achieve cross-traditional rational dialogue. + +### The Problem of the Selfish Gene +If we are products of natural selection, and selection favours genes that promote reproductive fitness, then "human nature" is not a stable, rationally accessible guide to flourishing. The naturalistic programme of Foot and Hursthouse faces this challenge. + +## Further Reading + +- Annas, J. (2011). *Intelligent Virtue*. Oxford University Press. +- Crisp, R. and Slote, M. (eds.) (1997). *Virtue Ethics*. Oxford University Press. +- Williams, B. (1985). *Ethics and the Limits of Philosophy*. Fontana. diff --git a/sample-sites/modern-philosophy/pages/eth-05-applied.md b/sample-sites/modern-philosophy/pages/eth-05-applied.md new file mode 100644 index 0000000..2a5894a --- /dev/null +++ b/sample-sites/modern-philosophy/pages/eth-05-applied.md @@ -0,0 +1,72 @@ +--- +title: Applied Ethics +sort: 140 +section-id: ethics +description: Ethical theory in practice โ€” bioethics, AI ethics, environmental ethics, and the methodology of applying philosophical principles to real-world problems. +language: en +--- + +# Applied Ethics + +Applied ethics brings philosophical theory to bear on concrete moral problems โ€” questions arising in medicine, technology, environmental policy, business, and law. The movement gained momentum in the 1960s and 1970s, driven partly by rapid advances in medicine and biotechnology that created novel moral dilemmas for which traditional frameworks offered insufficient guidance. + +## The Methodology of Applied Ethics + +Applied ethics is not mere application of theory to cases, as if ethical systems were algorithms waiting to be run. Several methodological approaches exist: + +**Top-down application**: Begin with an ethical theory (utilitarian, Kantian), derive principles, apply to cases. The limitation is that theories are contested; disagreement about foundations propagates into applied questions. + +**Case-based reasoning (casuistry)**: Begin with clear, paradigm cases where moral judgement is confident, then reason analogically to harder cases. Associated with clinical ethics consultation. The limitation is that paradigm cases must eventually be justified by something more than intuition. + +**Reflective equilibrium**: Move iteratively between principles and case judgements, revising both until achieving coherence. The approach most widely used in practice. + +**Specification**: Take general principles (do not harm, respect autonomy) and progressively specify them to handle particular cases without derivation from a complete theory. + +## Bioethics + +Bioethics addresses moral questions arising in medicine and biological research. The *Georgetown mantra* โ€” four principles identified by Beauchamp and Childress โ€” has become the dominant framework in clinical practice:^[Beauchamp, T. and Childress, J. (2019). *Principles of Biomedical Ethics*, 8th ed. Oxford University Press.] + +1. **Autonomy** โ€” Respect for the patient's self-determination. Informed consent is the operational expression of this principle. Patients with decision-making capacity may refuse treatment, even life-saving treatment. +2. **Beneficence** โ€” Act in the patient's best interests. Not merely "do no harm" but actively promote welfare. +3. **Non-maleficence** โ€” *Primum non nocere* (first, do no harm). Avoid imposing risks disproportionate to benefits. +4. **Justice** โ€” Fair distribution of benefits, risks, and burdens. Includes both procedural fairness and distributive justice in healthcare resource allocation. + +### End-of-Life Ethics + +The moral permissibility of assisted dying โ€” physician-assisted suicide (PAS) and voluntary euthanasia โ€” is among the most contested issues in bioethics. Arguments in favour appeal to autonomy: competent patients should determine the manner and timing of their deaths. Arguments against cite concerns about palliative care, the potential for coercion, and the symbolic significance of medical killing for the doctor-patient relationship. + +Peter Singer and James Rachels defend active euthanasia, arguing that the distinction between killing and letting die is morally irrelevant when intentions and outcomes are the same.^[Rachels, J. (1975). "Active and Passive Euthanasia." *New England Journal of Medicine*, 292(2).] Opponents invoke the doctrine of double effect and the integrity of the medical profession. + +### Research Ethics + +The Nuremberg Code (1947) and the Declaration of Helsinki (1964) emerged from scandals of medical experimentation on non-consenting subjects. Core requirements: voluntary informed consent, scientific validity, favourable risk-benefit ratio, and independent ethical review. The Belmont Report (1979) added justice as a requirement โ€” the burdens and benefits of research must be distributed fairly. + +## AI and Technology Ethics + +The rapid development of artificial intelligence creates a new domain for applied ethics. Key issues include: + +**Algorithmic bias**: Machine learning systems trained on historical data can encode and amplify existing discriminatory patterns. A loan approval algorithm trained on historical lending data may perpetuate racial discrimination without any discriminatory intent. Fairness criteria (demographic parity, equalised odds, calibration) are mathematically incompatible in general โ€” we cannot satisfy all simultaneously.^[Chouldechova, A. (2017). "Fair Prediction with Disparate Impact." *Big Data*, 5(2).] + +**Autonomous systems**: When an autonomous vehicle must choose between killing one pedestrian or five, how should it be programmed? Trolley-problem style dilemmas in algorithmic form raise questions about whether utilitarian calculus should be codified into machines, and who bears moral responsibility for automated decisions. + +**AI consciousness and moral status**: If future AI systems develop something like sentience or interests, do they merit moral consideration? The philosophical difficulty of consciousness (see the chapter on philosophy of mind) makes this question genuinely hard. + +**Privacy and surveillance**: The collection of vast personal data by states and corporations raises questions about informational privacy as an aspect of autonomy and dignity. Nissenbaum's concept of *contextual integrity* โ€” information flows respect privacy when they match the norms of the context in which they were generated โ€” provides a useful framework. + +## Environmental Ethics + +Traditional ethics is anthropocentric โ€” it recognises moral obligations only to persons. Environmental ethics asks: do non-human animals, species, ecosystems, or nature as a whole have intrinsic moral value? + +**Animal ethics**: Peter Singer's utilitarian case for animal liberation grounds obligations in the capacity for suffering: if pain is bad for humans, it is bad for pigs, who suffer equally.^[Singer, P. (1975). *Animal Liberation*. New York Review Books.] Tom Regan's rights-based account argues that animals who are "subjects of a life" โ€” with beliefs, desires, and a welfare โ€” have inherent value that may not be traded off against aggregate utility.^[Regan, T. (1983). *The Case for Animal Rights*. University of California Press.] + +**Biocentric ethics** (Paul Taylor): Every living organism has a good of its own that commands moral respect. We have prima facie duties not to harm living things, override-able only for weighty reasons.^[Taylor, P. (1986). *Respect for Nature*. Princeton University Press.] + +**Ecocentric ethics**: Aldo Leopold's *land ethic* โ€” "A thing is right when it tends to preserve the integrity, stability, and beauty of the biotic community" โ€” extends moral consideration to ecosystems and species, not just individuals.^[Leopold, A. (1949). *A Sand County Almanac*. Oxford University Press.] + +**Climate ethics** raises questions about intergenerational justice (obligations to future persons), international justice (who bears the costs of mitigation), and the ethics of geoengineering. + +## Further Reading + +- Rachels, J. and Rachels, S. (2019). *The Elements of Moral Philosophy*, 9th ed. McGraw-Hill. +- Jamieson, D. (2014). *Reason in a Dark Time: Why the Struggle Against Climate Change Failed*. Oxford University Press. +- Floridi, L. (ed.) (2015). *The Onlife Manifesto*. Springer. diff --git a/sample-sites/modern-philosophy/pages/eth-06-political.md b/sample-sites/modern-philosophy/pages/eth-06-political.md new file mode 100644 index 0000000..5bdc0a0 --- /dev/null +++ b/sample-sites/modern-philosophy/pages/eth-06-political.md @@ -0,0 +1,73 @@ +--- +title: Political Philosophy +sort: 150 +section-id: ethics +description: Rawls, Nozick, communitarianism, and contemporary debates about justice, liberty, and the legitimate authority of the state. +language: en +--- + +# Political Philosophy + +Political philosophy investigates the normative foundations of political institutions: the state, law, political authority, rights, and justice. Its central questions include: What justifies political authority? What makes a distribution of benefits and burdens just? What are the limits of individual liberty? What do citizens owe one another? + +## The Social Contract Tradition + +The dominant tradition in modern political philosophy grounds political authority in a *social contract* โ€” an actual or hypothetical agreement among individuals to establish political institutions. The tradition includes Hobbes, Locke, and Rousseau, but it is John Rawls who gave it its most sophisticated contemporary form. + +### Hobbes +Thomas Hobbes (1651) argued that without political authority, life would be "solitary, poor, nasty, brutish, and short."^[Hobbes, T. (1651). *Leviathan*, Ch. XIII.] Rational agents in the state of nature would contract into an absolute sovereign to secure peace. Hobbes's argument is primarily consequentialist in structure: sovereignty is justified by the order it creates. + +### Locke +John Locke (1689) grounded political authority in natural rights โ€” rights to life, liberty, and property that individuals possess prior to and independently of the state. Government is legitimate only if it protects these rights; when it systematically violates them, citizens have a right of revolution. Locke's theory provides the philosophical basis for liberal constitutionalism.^[Locke, J. (1689). *Two Treatises of Government*, Second Treatise.] + +## Rawls's Theory of Justice + +John Rawls (*A Theory of Justice*, 1971) represents the most influential work in twentieth-century political philosophy. Rawls aims to identify principles of justice that free and rational persons would accept in an original position of equality. + +### The Original Position and the Veil of Ignorance + +The *original position* is a hypothetical decision procedure. We imagine choosing principles of justice behind a *veil of ignorance*: we do not know our place in society, our class position, our natural abilities, our conception of the good, or the generation we belong to. This ensures impartiality โ€” no one can tailor principles to benefit their particular position.^[Rawls, J. (1971). *A Theory of Justice*. Harvard University Press, ยง3.] + +From behind the veil, Rawls argues, rational agents would choose two principles: + +1. **The Equal Liberty Principle**: Each person is to have an equal right to the most extensive system of equal basic liberties compatible with a similar system of liberty for all. +2. **The Difference Principle**: Social and economic inequalities are to be arranged so that they are: (a) attached to offices and positions open to all under fair equality of opportunity, and (b) to the greatest benefit of the least advantaged members of society. + +The principles are *lexically ordered*: the first has absolute priority over the second. Basic liberties cannot be traded off against economic gains. + +### The Difference Principle + +The Difference Principle is Rawls's most distinctive and controversial contribution. It permits inequalities only if they maximally benefit the worst-off group. This *maximin* strategy โ€” maximise the minimum position โ€” is what rational agents under uncertainty would choose, according to Rawls. + +The argument: since I do not know whether I will be advantaged or disadvantaged, and since the stakes (basic life prospects) are very high, rationality demands choosing the arrangement that makes the worst-case scenario as good as possible. + +## Nozick's Libertarianism + +Robert Nozick (*Anarchy, State, and Utopia*, 1974) offers a forceful alternative.^[Nozick, R. (1974). *Anarchy, State, and Utopia*. Basic Books.] Beginning from strong Lockean natural rights โ€” that individuals may not be used against their will as means to others' ends โ€” Nozick argues that only a minimal state (limited to protecting against force and fraud) is justified. + +Any more extensive state violates individual rights. Redistributive taxation, for Nozick, is morally equivalent to forced labour: it takes the product of a person's labour and transfers it to others without consent. + +Nozick's *entitlement theory* of justice: a distribution is just if it arises from just original acquisitions and just transfers. Historical process, not distributional pattern, determines justice. No patterned principle โ€” whether egalitarian or utilitarian โ€” can be maintained without continuous interference with free exchange. + +### Wilt Chamberlain Argument +Nozick's celebrated argument: suppose we start from any distribution D1 you consider just. Wilt Chamberlain, a basketball star, charges fans 25 cents per game to see him play. One million fans freely pay. Now Chamberlain has $250,000 more than D1 allowed. Is D2 unjust? But it arose through voluntary exchanges from a just starting point. Any patterned theory must continuously prohibit voluntary exchanges โ€” and this is incompatible with liberty. + +## Communitarianism + +In the 1980s, a group of philosophers โ€” Michael Sandel, Charles Taylor, Alasdair MacIntyre, and Michael Walzer โ€” challenged liberalism's conception of the self and the priority it gives to justice over the good. + +**Sandel's critique of the unencumbered self**: Rawls's original position presupposes a self that is prior to and independent of its ends and social roles. But this is incoherent: we cannot abstract ourselves from the constitutive attachments and community memberships that make us who we are. A more adequate political philosophy would recognise that we are *embedded* in communities whose values and traditions define our identities.^[Sandel, M. (1982). *Liberalism and the Limits of Justice*. Cambridge University Press.] + +**Walzer's complex equality**: Justice requires that different social goods โ€” money, political power, medical care, education โ€” be distributed according to their own internal norms, not reduced to a single metric. Injustice is not inequality per se but *dominance*: when one good (typically money) is used to control access to all others.^[Walzer, M. (1983). *Spheres of Justice*. Basic Books.] + +## Global Justice + +Rawls's *The Law of Peoples* (1999) applied his framework internationally, but controversially limited global distributive obligations to "duty of assistance" to "burdened societies." Cosmopolitan theorists (Thomas Pogge, Charles Beitz) argue that global economic institutions impose injustice on the world's poor, generating stringent obligations to reform them.^[Pogge, T. (2002). *World Poverty and Human Rights*. Polity Press.] + +The debate between Rawlsian nationalism and cosmopolitanism turns on whether Rawlsian principles apply only within cooperative schemes (nation-states) or to all human beings as such. + +## Further Reading + +- Freeman, S. (2007). *Justice and the Social Contract*. Oxford University Press. +- Cohen, G.A. (2008). *Rescuing Justice and Equality*. Harvard University Press. +- Kymlicka, W. (2002). *Contemporary Political Philosophy*, 2nd ed. Oxford University Press. diff --git a/sample-sites/modern-philosophy/pages/further-reading.md b/sample-sites/modern-philosophy/pages/further-reading.md new file mode 100644 index 0000000..1715636 --- /dev/null +++ b/sample-sites/modern-philosophy/pages/further-reading.md @@ -0,0 +1,136 @@ +--- +title: Further Reading +sort: 110 +section-id: conclusion +description: Annotated bibliography organised by chapter, with commentary on essential secondary texts and resources for continued study. +language: en +--- + +# Further Reading + +This annotated bibliography is organised by chapter. For each topic, the most accessible introductory texts are listed first, followed by more advanced or specialised works. All items marked **[Core]** are considered essential reading; unmarked items represent productive next steps for those wishing to go deeper. + +--- + +## Part I: Epistemology + +### What is Knowledge? +**[Core]** Gettier, E.L. (1963). "Is Justified True Belief Knowledge?" *Analysis*, 23(6), 121โ€“123. โ€” Three pages that changed epistemology. Required reading. + +**[Core]** Chisholm, R. (1977). *Theory of Knowledge*, 2nd ed. Prentice Hall. โ€” The classic textbook on epistemological foundations. + +Zagzebski, L. (1994). "The Inescapability of Gettier Problems." *Philosophical Quarterly*, 44(174), 65โ€“73. โ€” Shows the structural depth of the problem. + +Williamson, T. (2000). *Knowledge and Its Limits*. Oxford University Press. โ€” Defends knowledge as a prime epistemic concept; difficult but rewarding. + +### Perception and Reality +**[Core]** Ayer, A.J. (1956). *The Problem of Knowledge*. Penguin. โ€” Accessible and wide-ranging. + +Dancy, J. (1985). *Introduction to Contemporary Epistemology*. Blackwell. โ€” Chapter 6 on perception is particularly good. + +McDowell, J. (1994). *Mind and World*. Harvard University Press. โ€” Demanding but essential for understanding the conceptualism debate. + +### Scepticism +**[Core]** Descartes, R. (1641). *Meditations on First Philosophy*. โ€” The primary source; any good translation suffices. + +Stroud, B. (1984). *The Significance of Philosophical Scepticism*. Oxford University Press. โ€” Why scepticism cannot be easily dismissed. + +DeRose, K. (2009). *The Case for Contextualism*. Oxford University Press. + +### Truth +**[Core]** Horwich, P. (1998). *Truth*, 2nd ed. Oxford University Press. โ€” Deflationary theory, clearly argued. + +Lynch, M. (2009). *Truth as One and Many*. Oxford University Press. โ€” Pluralist theory. + +--- + +## Part II: Metaphysics + +### Existence and Ontology +**[Core]** Quine, W.V.O. (1948). "On What There Is." *Review of Metaphysics*, 2(5). โ€” The classic statement of Quinean ontology. + +**[Core]** van Inwagen, P. (1998). "Meta-Ontology." *Erkenntnis*, 48(2โ€“3), 233โ€“250. + +Thomasson, A. (2015). *Ontology Made Easy*. Oxford University Press. โ€” Deflationary approach; valuable counterpoint to heavyweight ontology. + +### Identity and Persistence +**[Core]** Parfit, D. (1984). *Reasons and Persons*, Part III. Oxford University Press. โ€” The most influential modern treatment. + +Lewis, D. (1976). "Survival and Identity." In Rorty, A. (ed.), *The Identities of Persons*. Berkeley. + +Olson, E. (1997). *The Human Animal*. Oxford University Press. โ€” Animalist view. + +### Free Will +**[Core]** Kane, R. (1996). *The Significance of Free Will*. Oxford University Press. โ€” Best defence of libertarianism. + +**[Core]** Frankfurt, H. (1969). "Alternate Possibilities and Moral Responsibility." *Journal of Philosophy*, 66(23). โ€” Frankfurt cases; only five pages, transformative. + +Strawson, P.F. (1962). "Freedom and Resentment." *Proceedings of the British Academy*, 48. โ€” Foundational compatibilist paper. + +Fischer, J.M. and Ravizza, M. (1998). *Responsibility and Control*. Cambridge University Press. + +### Philosophy of Mind +**[Core]** Nagel, T. (1974). "What Is It Like to Be a Bat?" *Philosophical Review*, 83(4). โ€” The classic statement of the explanatory gap. + +**[Core]** Chalmers, D. (1996). *The Conscious Mind*. Oxford University Press. โ€” Comprehensive case for the hard problem. + +Dennett, D. (1991). *Consciousness Explained*. Little, Brown. โ€” The physicalist response; readable and provocative. + +Jackson, F. (1986). "What Mary Didn't Know." *Journal of Philosophy*, 83(5). โ€” Knowledge argument in five pages. + +--- + +## Part III: Ethics + +### Metaethics +**[Core]** Mackie, J.L. (1977). *Ethics: Inventing Right and Wrong*. Penguin. โ€” Error theory; accessible and well-argued. + +Blackburn, S. (1998). *Ruling Passions*. Oxford University Press. โ€” Best recent defence of quasi-realism. + +Enoch, D. (2011). *Taking Morality Seriously*. Oxford University Press. โ€” Strong defence of robust moral realism. + +### Consequentialism +**[Core]** Mill, J.S. (1863). *Utilitarianism*. โ€” Short; read in an afternoon; annotated editions recommended. + +**[Core]** Singer, P. (1979). *Practical Ethics*. Cambridge University Press. โ€” Applies utilitarian reasoning to live issues. + +Parfit, D. (1984). *Reasons and Persons*, Part IV. โ€” "Repugnant Conclusion" and population ethics; essential. + +### Deontological Ethics +**[Core]** Kant, I. (1785). *Groundwork of the Metaphysics of Morals*. Trans. Korsgaard. Cambridge UP. โ€” Use Korsgaard's translation and commentary. + +**[Core]** Ross, W.D. (1930). *The Right and the Good*, Chs. 1โ€“2. Oxford University Press. + +Scanlon, T.M. (1998). *What We Owe to Each Other*. Harvard University Press. โ€” Contractualist deontology; rich and rewarding. + +### Virtue Ethics +**[Core]** Aristotle. *Nicomachean Ethics*, Books Iโ€“II, X. โ€” Any good translation. + +**[Core]** Hursthouse, R. (1999). *On Virtue Ethics*. Oxford University Press. + +MacIntyre, A. (1981). *After Virtue*. University of Notre Dame Press. โ€” Polemical and influential. + +### Political Philosophy +**[Core]** Rawls, J. (1971). *A Theory of Justice*. Harvard University Press. โ€” Read at minimum Part I. + +**[Core]** Nozick, R. (1974). *Anarchy, State, and Utopia*. Basic Books. โ€” Especially Ch. 7 on distributive justice. + +Kymlicka, W. (2002). *Contemporary Political Philosophy*, 2nd ed. Oxford University Press. โ€” Best survey text. + +--- + +## General Philosophy Reference + +**Stanford Encyclopedia of Philosophy** (plato.stanford.edu) โ€” Freely available online; peer-reviewed, regularly updated. An invaluable first resource for any philosophical topic. + +**[Core]** Blackburn, S. (1996). *Oxford Dictionary of Philosophy*, 3rd ed. Oxford University Press. โ€” Concise, reliable reference for key terms. + +Craig, E. (ed.) (1998). *Routledge Encyclopedia of Philosophy*. โ€” 10-volume scholarly reference. + +--- + +## On Reading Philosophy + +Philosophical texts reward rereading. A text that seems clear at first glance often conceals assumptions that become visible only on the third or fourth reading. Keep a running list of assumptions, note where each argument depends on undefended premises, and always ask: what would need to be true for this argument to fail? + +Discussion and disagreement are essential. Read with a philosophical friend or in a seminar. The objections you make and receive in conversation will teach you more than any further reading. diff --git a/sample-sites/modern-philosophy/pages/how-to-use.md b/sample-sites/modern-philosophy/pages/how-to-use.md new file mode 100644 index 0000000..ba21a7b --- /dev/null +++ b/sample-sites/modern-philosophy/pages/how-to-use.md @@ -0,0 +1,63 @@ +--- +title: How to Use This Book +sort: 110 +section-id: front-matter +description: Reading guide, chapter dependencies, glossary note, and further reading approach. +language: en +--- + +![A well-stocked research library](assets/images/library.jpg) + +# How to Use This Book + +This book is designed to be accessible to readers coming from different starting points and with different purposes. What follows is a brief guide to getting the most from it. + +## Sequence and Structure + +The book is organised into three parts โ€” Epistemology, Metaphysics, and Ethics โ€” each comprising six chapters. The parts are designed to be read in order, since later discussions often presuppose earlier material. The epistemology chapters, for instance, establish vocabulary and conceptual distinctions (justification, a priori knowledge, reliabilism, coherentism) that recur throughout the metaphysics and ethics discussions. + +Within each part, the chapters proceed from foundational questions toward more specific and applied ones. In epistemology, we begin with the basic analysis of knowledge before examining specific faculties (perception, reason) and specific challenges (scepticism). In ethics, we examine the foundations of moral inquiry before turning to the major normative theories and then applied questions. + +That said, the book is cross-referenced and many chapters can be read independently by a reader who already has some background. The chapter on free will and determinism (Part II, Chapter 4) can be read alongside the epistemology chapter on scepticism and the ethics chapter on moral responsibility, and is usefully paired with the applied ethics chapter's discussion of criminal justice. The chapter on philosophy of mind (Part II, Chapter 5) is closely connected to the epistemology discussion of perception and knowledge. + +## Chapter Structure + +Each chapter follows a common pattern: + +1. **Introduction** โ€” orienting the problem, explaining why it matters +2. **Main positions** โ€” surveyed with their central arguments +3. **Key arguments and objections** โ€” examined with care +4. **Connections and implications** โ€” how the debate bears on other questions +5. **Inline footnotes** โ€” key citations in the form `^[Author, Year, p.X]` + +I have used inline footnotes throughout rather than endnotes. Philosophy students need to learn to read with citations โ€” to understand that positions have sources, that arguments have authors, and that intellectual honesty requires acknowledging where ideas come from. + +## The Primary Sources + +This is an introductory text. It cannot substitute for reading primary sources, and it is not intended to. The goal is to prepare you to read Descartes, Hume, Kant, and their successors with understanding โ€” to give you the context and vocabulary to follow their arguments, so that when you turn to the *Meditations* or the *Enquiry* or the *Groundwork*, you know what question you are supposed to be thinking about. + +At the end of the book you will find an annotated bibliography organised by chapter. Each entry includes a note on what the text contributes and who it is most suitable for. Use it as a starting point for primary reading, not as a substitute for it. + +## A Note on Difficulty + +Philosophy is hard. It requires holding complex arguments in memory while evaluating their steps, tracking distinctions that initially seem subtle and become crucial, and maintaining intellectual patience through periods of genuine uncertainty. This is not a reason to find the difficulty discouraging โ€” it is a reason to take it seriously. + +When you find yourself confused, the first question to ask is not "Am I misunderstanding the argument?" but "Is the argument actually this hard?" Sometimes the answer to the second question is yes, and the apparent confusion reflects something genuinely difficult in the material. At other times, a re-reading or a change of perspective resolves things. + +It is worth keeping notes as you read โ€” writing down the main claim of each section, the central argument, and your own questions and objections. Philosophy is better done with a pen in hand than as a purely passive activity. + +## On Philosophical Writing + +Students who will be writing essays in philosophy should note that philosophical writing is argument, not assertion. The task is not to state what you believe but to give reasons for believing it, to consider and respond to objections, and to be honest about what you do not yet know. + +Good philosophical writing is clear, precise, and fair to opposing views. It uses technical vocabulary accurately โ€” not to impress, but because precision matters and ordinary language is often insufficiently precise for philosophical work. It acknowledges the difficulty of the questions rather than pretending to more certainty than is warranted. + +These are not stylistic preferences. They are requirements of intellectual honesty, and they are what distinguish good philosophical writing from merely asserting things confidently. + +## Glossary + +Technical terms are defined when first introduced and collected in the index with page references. Key terms include: *a priori*, *a posteriori*, *analytic*, *synthetic*, *justified true belief*, *internalism*, *externalism*, *substance*, *property*, *supervenience*, *compatibilism*, *libertarianism* (in the metaphysical sense), *hard determinism*, *consequentialism*, *deontology*, *virtue ethics*, *metaethics*, *normative ethics*. + +Do not be discouraged if these terms initially seem arbitrary. They acquire meaning through the arguments that use them, and they become tools rather than obstacles once you have enough context to understand why the distinctions they mark are important. + +Good reading. diff --git a/sample-sites/modern-philosophy/pages/meta-01-existence.md b/sample-sites/modern-philosophy/pages/meta-01-existence.md new file mode 100644 index 0000000..3500daa --- /dev/null +++ b/sample-sites/modern-philosophy/pages/meta-01-existence.md @@ -0,0 +1,55 @@ +--- +title: Existence and Being +sort: 100 +section-id: metaphysics +description: Ontology basics, Quine's criterion of ontological commitment, existence as a predicate, and Meinong's jungle. +language: en +--- + +# Existence and Being + +Metaphysics is the study of the most fundamental features of reality โ€” what exists, what kinds of things exist, and what it is for something to exist at all. Ontology, its central division, addresses the question: *what is there?* + +The question sounds trivially answerable: there is everything there is, and nothing else. Quine gave this answer in a sentence: "To be is to be the value of a variable" ^[Quine, W.V.O., "On What There Is", *Review of Metaphysics* 2, 1948]. But behind this slogan lies a substantial and contested philosophical methodology. + +## Quine's Criterion of Ontological Commitment + +Quine argued that the ontological commitments of a theory are revealed by *regimentation* โ€” translating the theory into first-order predicate logic and examining what the variables in the existential quantifiers must range over ^[Quine, W.V.O., "On What There Is"; see also *Word and Object*, MIT Press, 1960, ch.7]. + +If a true theory says "There are prime numbers between 10 and 20," and the best regimentation of this claim quantifies over numbers, then the theory is committed to the existence of numbers. One cannot simultaneously assert a theory and deny the existence of entities the theory quantifies over. + +The Quinean approach transformed ontology from a speculative metaphysical exercise into a semi-technical discipline: the question "Does X exist?" becomes "Does our best overall theory of the world require quantifying over Xs?" This is ontology anchored to science. + +**The criterion of ontological commitment** is: a theory is ontologically committed to those entities that, when the theory is put in canonical notation, the bound variables must range over if the theory is true. + +## Existence as a Predicate + +Can existence be predicated of individuals? Kant famously argued that existence is not a real predicate โ€” it adds nothing to the concept of a thing ^[Kant, I., *Critique of Pure Reason*, A598/B626]. "God is omnipotent" attributes a property; "God exists" does not attribute a property but rather asserts that the concept of God is instantiated. + +This view is embedded in the standard logical treatment: in first-order predicate logic, existence is expressed by the existential quantifier (โˆƒx), not by a predicate. "Tigers exist" becomes "there is at least one thing that is a tiger," not "tigers have the property of existence." + +If existence is not a predicate, the ontological argument for God's existence โ€” which treats existence as a perfection that maximally great beings must have โ€” fails at the point of treating existence as a property that can be possessed in greater or lesser degree. + +## Meinong and Non-Existent Objects + +Alexius Meinong argued that the domain of objects is wider than the domain of existents ^[Meinong, A., "On the Theory of Objects", 1904]. There are objects โ€” the golden mountain, the round square, Sherlock Holmes โ€” that do not exist. These non-existent objects nonetheless have properties: the golden mountain is golden and mountainous; the round square is both round and square (and therefore impossible); Sherlock Holmes lived at 221B Baker Street. + +Meinong's *Theory of Objects* distinguishes *existence* (the mode of being of concrete things), *subsistence* (the mode of being of abstract objects like numbers and propositions), and mere *Sosein* (having a nature, without any mode of being). Non-existent objects have Sosein without existence. + +Russell objected vigorously to this "Meinongian jungle" of non-existent objects, calling it "a failure of that robust sense of reality which ought to be preserved even in the most abstract studies" ^[Russell, B., "Review of Meinong", *Mind* 14, 1905, p.533]. His theory of definite descriptions was designed to handle sentences about non-existents without ontological commitment to them. + +## Russell's Theory of Descriptions + +Russell's theory of descriptions analyses sentences of the form "The F is G" as: there is exactly one F, and that F is G ^[Russell, B., "On Denoting", *Mind* 14, 1905, pp.479-493]. "The present King of France is bald" is false because there is no present King of France โ€” the uniqueness condition fails. We do not need to posit a non-existent King of France; we only need to recognise that the sentence has a false existential presupposition. + +This "logical paraphrase" strategy โ€” finding analyses of problematic sentences that avoid commitment to suspect entities โ€” became a central tool of analytic philosophy. Ockham's razor ("Do not multiply entities beyond necessity") is the methodological principle: ontological economy is a theoretical virtue. + +## Contemporary Debates + +**Metaontology** โ€” the study of what ontological questions mean and how they should be answered โ€” has become a major area. Carnap distinguished *internal* questions (do numbers exist, within the mathematical framework?) from *external* questions (does the mathematical framework correspond to reality?) and argued that external questions are pragmatic, not factual ^[Carnap, R., "Empiricism, Semantics and Ontology", *Revue Internationale de Philosophie* 4, 1950]. Quine rejected this distinction; contemporary metaontologists debate whether it can be rehabilitated. + +**Ontological pluralism** โ€” the view that existence itself is not univocal, but that there are different "modes of being" โ€” has been defended by Kris McDaniel and others, reviving something like the Meinongian project with better tools ^[McDaniel, K., *The Fragmentation of Being*, Oxford UP, 2017]. + +**Truthmaker theory** holds that truths are made true by features of reality, and asks what those features are for different classes of truths โ€” mathematical truths, modal truths, moral truths ^[Armstrong, D.M., *Truth and Truthmakers*, Cambridge UP, 2004]. + +Existence and being may seem like the most abstract possible questions. But they have concrete implications: whether numbers exist bears on the foundations of mathematics; whether moral properties exist bears on the nature of moral knowledge; whether merely possible objects exist bears on modal semantics. Ontology is, as Quine put it, where logic and the world meet. diff --git a/sample-sites/modern-philosophy/pages/meta-02-identity.md b/sample-sites/modern-philosophy/pages/meta-02-identity.md new file mode 100644 index 0000000..5f67141 --- /dev/null +++ b/sample-sites/modern-philosophy/pages/meta-02-identity.md @@ -0,0 +1,55 @@ +--- +title: Identity and Persistence +sort: 110 +section-id: metaphysics +description: The Ship of Theseus, personal identity, and four-dimensionalist theories of persistence. +language: en +--- + +# Identity and Persistence + +What makes a thing the same thing over time? The ship that returned to Athens was rebuilt plank by plank during the voyage; no original material remained. Is it the same ship? The person who wakes tomorrow morning shares your memories and psychology โ€” but not your matter, since your cells replace themselves over years. Are they you? + +These are not idle puzzles. They bear on personal survival, moral responsibility, and the metaphysics of change, and their answers connect to fundamental questions about what kinds of things exist. + +## The Ship of Theseus + +The Ship of Theseus is one of philosophy's oldest thought experiments, recorded by Plutarch ^[Plutarch, *Theseus*, ch.23, c.75 CE]. Its philosophical interest lies not in the historical case but in the structure of the puzzle it reveals: ordinary objects persist through gradual material change, but taken to the limit, material continuity seems to dissolve. + +Hobbes added a further twist: suppose someone keeps all the original planks and reassembles them. Which is the original ship โ€” the continuously maintained vessel or the reconstructed one? ^[Hobbes, T., *De Corpore*, 1655, II.xi.7]. The puzzle reveals that "same ship" may be determined by different identity criteria (material continuity, spatial-temporal continuity, functional continuity) that can diverge. + +This connects to a broader metaphysical debate about *constitution*: when a lump of clay is shaped into a statue, are there two objects (the lump and the statue) that share matter but have different persistence conditions? Or only one? Defenders of constitution theory say two; critics say one (and find the view of two things in the same place implausible) ^[Wiggins, D., *Sameness and Substance*, Blackwell, 1980; Gibbard, A., "Contingent Identity", *Journal of Philosophical Logic* 4, 1975]. + +## Personal Identity: The Classical View + +John Locke offered the first systematic philosophical analysis of personal identity ^[Locke, J., *Essay*, II.xxvii]. He distinguished the identity of *substance* (a material thing, continuous in matter), *organism* (a living thing, continuous in life), and *person* (a thinking conscious being, continuous in *consciousness*). + +Persons, for Locke, persist through *psychological continuity* โ€” specifically, memory. What makes the person who did action A the same person as me is my memory of having done A. This allows persons to come apart from both bodies and souls: the prince and the cobbler might "swap" if their consciousnesses were somehow exchanged. + +**Objections:** Bishop Butler accused Locke of circularity: memory presupposes personal identity, so it cannot constitute it ^[Butler, J., *The Analogy of Religion*, 1736, Appendix I]. Thomas Reid's *brave officer paradox* sharpened this: an old general remembers his younger self's bravery as a junior officer, but the officer (flogged as a boy) no longer remembers the boy's actions. By transitivity, the general is not the same person as the boy โ€” but this seems absurd ^[Reid, T., *Essays on the Intellectual Powers of Man*, 1785, III.6]. + +## Neo-Lockean Theories + +Derek Parfit developed the most sophisticated neo-Lockean account ^[Parfit, D., *Reasons and Persons*, Oxford UP, 1984, Part III]. He replaced memory with *psychological continuity* โ€” overlapping chains of psychological connections (memories, intentions, beliefs, desires) โ€” and argued that what matters for survival is not strict identity but this continuity relation. + +Parfit's striking conclusion: personal identity is not what matters in survival. In cases of fission (where your psychology is duplicated in two people), neither resulting person is strictly you โ€” but both have what matters as much as survival does. We should care about psychological continuity and connectedness, not about identity itself. + +This has radical implications for ethics: if personal identity does not matter in itself, many concerns about future persons โ€” including concerns about one's own future self โ€” are impersonal, and the boundaries between persons may be less sharp than we normally assume. + +## The Biological Criterion + +Eric Olson has argued for *animalism*: we are human animals, and our persistence conditions are those of biological organisms ^[Olson, E., *The Human Animal*, Oxford UP, 1997]. Personal identity is not constituted by psychological continuity; you persist as long as your body's metabolism continues. Brain transplants, on this view, are body transplants. + +Animalism avoids neo-Lockean puzzles by refusing to split the person from the organism, but it faces difficulties with cerebral bisection, personal survival, and cases where psychological continuity intuitively tracks identity better than biological continuity does. + +## Four-Dimensionalism + +The *four-dimensionalist* (or *perdurantist*) view holds that objects persist by having temporal parts at different times, just as they have spatial parts at different locations ^[Lewis, D., *On the Plurality of Worlds*, Blackwell, 1986, pp.202-204; Sider, T., *Four-Dimensionalism*, Oxford UP, 2001]. The person who existed yesterday is a *temporal part* of a four-dimensional object extended through time as well as space. + +On this view, there is no problem of persistence through change: different temporal parts can have different properties. The Ship of Theseus problem dissolves: the ship at time t1 and the ship at time t2 are different temporal parts of the same four-dimensional whole, even though their material composition differs. + +Four-dimensionalism is technically elegant but counterintuitive: it implies that we never change โ€” what we ordinarily call "my change" is two different temporal parts having different properties. And the multiplication of temporal parts raises concerns about parsimony. + +**Three-dimensionalists** (or *endurantists*) hold that ordinary objects persist by being *wholly present* at each moment of their existence. They accept genuine identity through time and must therefore give an account of how the same thing can have different properties at different times (through *temporally modified* property ascription, or relativisation to times). + +The persistence debate connects to the metaphysics of time: *presentists*, who hold only the present exists, are naturally endurantists; *eternalists*, who hold past, present, and future equally exist, can more naturally accommodate perdurance. diff --git a/sample-sites/modern-philosophy/pages/meta-03-causation.md b/sample-sites/modern-philosophy/pages/meta-03-causation.md new file mode 100644 index 0000000..a78d45f --- /dev/null +++ b/sample-sites/modern-philosophy/pages/meta-03-causation.md @@ -0,0 +1,61 @@ +--- +title: Causation +sort: 120 +section-id: metaphysics +description: Hume's regularity account of causation, counterfactual theories, mechanistic theories, and causal pluralism. +language: en +--- + +# Causation + +Causation is among the most pervasive features of reality and among the most philosophically contested. We invoke causal relations constantly: the window broke because it was struck by the ball; the fire spread because of the wind; the patient died because of the infection. Causal explanation, causal reasoning, and causal intervention underlie science, medicine, law, and everyday thought. Yet what causation *is* โ€” what it is for one event to cause another โ€” remains deeply controversial. + +## Hume's Regularity Theory + +Hume's analysis of causation, discussed in the context of empiricism (Chapter 4), remains the starting point for the contemporary debate. Hume distinguished two definitions of cause. + +The first, in terms of *constant conjunction*: "An object, followed by another, and where all the objects similar to the first are followed by objects similar to the second" ^[Hume, D., *Enquiry*, ยง7.2]. Causation, on this view, is nothing over and above regular succession: whenever an event of type A occurs, an event of type B follows. + +The second, in terms of *determination*: "An object followed by another, and whose appearance always conveys the thought to that other." This psychological definition reveals that the impression of necessary connection is in us, not in the objects. + +The *regularity theory* developed from Hume's first definition. Mill systematised it with his *methods of agreement, difference*, and *concomitant variation* for identifying causal regularities ^[Mill, J.S., *A System of Logic*, 1843, III.viii]. + +**Objections:** Mere regular succession does not suffice for causation. Day regularly precedes night, but dawn does not cause dusk. Common causes produce correlated effects โ€” thunder correlates with lightning, but neither causes the other. And regularities can be accidental (all gold spheres are smaller than the sun) rather than causal. + +## Counterfactual Theories + +David Lewis's *counterfactual theory* defines causation in terms of counterfactual dependence: C causes E if and only if, had C not occurred, E would not have occurred ^[Lewis, D., "Causation", *Journal of Philosophy* 70, 1973, pp.556-567]. + +This approach handles many cases better than regularity theories. The counterfactual "if the ball had not struck the window, the window would not have broken" is true (under normal conditions); hence the striking caused the breaking. Cases of accidental correlation are handled naturally: even if thunder and lightning are regularly correlated, it is not true that if thunder had not occurred, lightning would not have. + +**Problems:** *Preemption* โ€” where two potential causes compete and one "preempts" the other โ€” is difficult. If two assassins shoot simultaneously and one bullet arrives first, the first shot caused the death; but it is not clear that the death counterfactually depends on the first shot, since the second would have caused it anyway. Lewis developed increasingly complex responses involving *fragility*, *quasi-dependence*, and *influence* ^[Lewis, D., "Causation as Influence", *Journal of Philosophy* 97, 2000]. + +*Overdetermination* โ€” where two simultaneous causes each suffice for the effect โ€” creates parallel problems. + +## Mechanistic Theories + +*Mechanistic* or *process* theories hold that causation consists in the transmission of energy, momentum, or causal influence through a spatiotemporally continuous process ^[Salmon, W., *Scientific Explanation and the Causal Structure of the World*, Princeton UP, 1984; Dowe, P., *Physical Causation*, Cambridge UP, 2000]. + +Wesley Salmon proposed that causal processes are distinguished from *pseudo-processes* (like shadows) by their ability to transmit a *mark* โ€” an alteration made at one point that propagates forward. Phil Dowe replaced this with a conserved quantity account: a causal process is one that transmits a conserved quantity (energy, charge, momentum). + +Mechanistic theories have the advantage of closely tracking scientific practice โ€” physicists and biologists routinely explain by identifying mechanisms. But they face difficulties with causation by absence (the bridge collapsed *because* the engineers failed to inspect it), negative causation, and the causation of absences. + +## Interventionist Theories + +James Woodward developed an *interventionist* account, appealing to the notion of ideal intervention ^[Woodward, J., *Making Things Happen*, Oxford UP, 2003]. A variable X causes Y if there is a possible ideal intervention on X (one that changes X independently of other causes of Y) that changes Y. + +This connects causation to the notion of manipulation or control, and is particularly well-suited to the social and biological sciences. It captures the idea that causal claims are action-guiding: to know that X causes Y is to know that intervening on X will change Y. + +Interventionism faces the question of whether the interventionist account is circular โ€” since interventions are themselves causal notions. + +## Singular Causation and the Problem of Many Levels + +A recurring question: is causation a relation between *types* of events (event-type A regularly precedes event-type B) or between *tokens* โ€” particular, individual events (this striking caused this breaking)? + +Token causation matters for law: we want to know whether *this* person's negligence caused *this* accident, not whether negligence-type events generally precede accidents. + +The *problem of causal exclusion* (closely connected to philosophy of mind) asks how mental causation is possible if everything is determined at the physical level. If the physical causes of my action fully determine it, what work is left for my mental states to do? ^[Kim, J., *Mind in a Physical World*, MIT Press, 1998]. This challenges non-reductive physicalism about mind. + +**Causal pluralism** holds that there is no single analysis of causation that captures all uses of causal vocabulary ^[Hall, N., "Two Concepts of Causation", in *Causation and Counterfactuals*, ed. Collins et al., MIT Press, 2004]. There may be one concept of causation for physics, another for biology, another for the law. Pluralism is comfortable with this; the search for a unified account may be misconceived. + +The contemporary causation debate is technically sophisticated and connects to philosophy of science, philosophy of mind, and action theory. What is clear is that Hume was right about one thing: the concept of causation is not simply read off the surface of experience but requires serious philosophical analysis. diff --git a/sample-sites/modern-philosophy/pages/meta-04-freewill.md b/sample-sites/modern-philosophy/pages/meta-04-freewill.md new file mode 100644 index 0000000..0f2a4ed --- /dev/null +++ b/sample-sites/modern-philosophy/pages/meta-04-freewill.md @@ -0,0 +1,66 @@ +--- +title: Free Will and Determinism +sort: 130 +section-id: metaphysics +description: Hard determinism, libertarianism, compatibilism, and Frankfurt cases. +language: en +--- + +# Free Will and Determinism + +The free will debate is among philosophy's most enduring and personally consequential. It asks whether, in a deterministic universe, human beings can be genuinely free โ€” free in a way that makes praise, blame, punishment, and moral responsibility appropriate. The question matters not just theoretically but practically: legal systems, personal relationships, and our self-understanding all presuppose that people can be held responsible for their actions. + +## The Incompatibilist Intuition + +The *basic argument* for incompatibilism runs roughly as follows ^[Van Inwagen, P., *An Essay on Free Will*, Oxford UP, 1983, pp.56-105]: + +1. Determinism is true: every event, including every human action, is causally necessitated by prior events in conjunction with the laws of nature. +2. If determinism is true, then no one ever could have acted otherwise than they did. +3. Moral responsibility requires the ability to have acted otherwise. +4. Therefore, if determinism is true, no one is morally responsible for anything. + +This argument has considerable intuitive force. If my action was causally determined by events that happened before I was born, in what sense was it *my* choice? If the complete causal history of the universe made my decision inevitable, how am I the author of it in any meaningful sense? + +## Hard Determinism + +*Hard determinists* accept incompatibilism and accept determinism, concluding that free will does not exist and moral responsibility must be radically revised or abandoned. + +Derk Pereboom has argued for *hard incompatibilism* with increasing sophistication ^[Pereboom, D., *Living Without Free Will*, Cambridge UP, 2001]. He accepts that moral luck undermines responsibility, that determinism (or indeterminism, for different reasons) threatens desert-based punishment, but argues that a meaningful life can be constructed without the reactive attitudes (blame, indignation, gratitude) that presuppose responsibility. + +The practical implications are significant: criminal punishment on retributive grounds is unjustified; therapeutic and quarantine-based reasons for incapacitation remain. Reactive attitudes would be gradually replaced by more forward-looking responses. + +## Libertarianism About Free Will + +*Libertarians* (in the metaphysical sense, entirely distinct from political libertarianism) accept incompatibilism but reject determinism, maintaining that free will requires โ€” and we have โ€” a form of causation that is not deterministic. + +Agent causation: Roderick Chisholm argued that free action requires *agent causation* โ€” a primitive, irreducible capacity of persons as agents to initiate causal chains, not wholly determined by prior events ^[Chisholm, R., "Human Freedom and the Self", in *Free Will*, ed. Watson, Oxford UP, 1982]. + +The *undetermined choice*: Robert Kane developed an account on which free will-exercising decisions occur at moments of *self-forming actions* โ€” quantum-indeterminate moments of genuine undeterminedness, where the agent's character and reasons could have produced either outcome ^[Kane, R., *The Significance of Free Will*, Oxford UP, 1996]. + +Libertarianism faces the *luck objection*: if my action was undetermined, then it seems random rather than free. An undetermined choice is one that even I could not have predicted from my own character and reasons โ€” which seems to undermine rather than secure my authorship of the action. + +## Compatibilism + +*Compatibilists* reject the third premise of the basic argument. Moral responsibility, they argue, does not require the ability to have done otherwise in the libertarian sense. What matters is whether the action was performed for the right kinds of reasons, whether the agent was responsive to reasons, whether the action was voluntary in the relevant sense. + +**Classical compatibilism:** Hume, and following him most analytic philosophers of the twentieth century, held that freedom is simply the ability to act in accordance with one's own desires, without external compulsion ^[Hume, D., *Enquiry*, ยง8]. Coercion, addiction, and phobia compromise freedom; determinism does not. + +**Hierarchical compatibilism:** Harry Frankfurt proposed that what matters for freedom is the *structure* of the agent's motivations ^[Frankfurt, H., "Freedom of the Will and the Concept of a Person", *Journal of Philosophy* 68, 1971, pp.5-20]. We have first-order desires (I want to smoke) and second-order desires (I want to want to smoke, or I want not to want to smoke). A free agent is one whose first-order desires align with their second-order volitions โ€” who acts from desires they endorse. This hierarchical structure distinguishes the wanton (who acts on whatever desire is strongest) from the autonomous agent. + +**Reasons-responsiveness:** John Martin Fischer and Mark Ravizza developed the view that free agency requires mechanisms that are *reasons-responsive* โ€” mechanisms that would have produced different choices had there been different reasons ^[Fischer, J.M. and Ravizza, M., *Responsibility and Control*, Cambridge UP, 1998]. You are responsible for actions that flow from your own reasons-responsive mechanisms. + +## Frankfurt Cases + +Harry Frankfurt's most influential contribution is the *Frankfurt case*, designed to challenge the *Principle of Alternative Possibilities* (PAP): a person is morally responsible for their action only if they could have acted otherwise. + +The structure: Black wants Jones to perform some action. Black has installed a mechanism in Jones's brain that will, if Jones shows any sign of not performing the action, intervene and ensure Jones performs it. In fact, Jones performs the action on his own, and the mechanism never activates. Jones could not have done otherwise (the mechanism would have prevented it). But, Frankfurt argues, Jones is clearly morally responsible ^[Frankfurt, H., "Alternate Possibilities and Moral Responsibility", *Journal of Philosophy* 66, 1969, pp.829-839]. + +If Frankfurt cases are sound, PAP is false, and the basic incompatibilist argument loses its third premise. The literature on Frankfurt cases is enormous: compatibilists have used them to argue that alternative possibilities are not required for responsibility; incompatibilists have argued that Frankfurt cases either fail to establish genuine alternative possibilities being closed off, or leave open a "flicker of freedom" ^[Kane, R., "Two Kinds of Incompatibilism", *Philosophy and Phenomenological Research* 50, 1989]. + +## Strawson's Reactive Attitudes + +P.F. Strawson argued that the debate misses what is most important about responsibility: our *reactive attitudes* ^[Strawson, P.F., "Freedom and Resentment", *Proceedings of the British Academy* 48, 1962]. Resentment, gratitude, indignation, and love are the appropriate responses to the quality of an agent's will toward us. These attitudes constitute โ€” rather than presuppose โ€” our moral practices. The question of whether determinism is true is largely beside the point; what matters is whether we are the kinds of creatures to whom it is appropriate to hold reactive attitudes, and we clearly are. + +Strawson's insight is that moral responsibility is essentially an interpersonal, practice-constituted phenomenon, not primarily a metaphysical one. + +The free will debate has not converged. Compatibilism is the most widely held view among professional philosophers, but libertarian and hard incompatibilist positions retain significant defenders. The question of what freedom requires โ€” and whether we have it โ€” remains genuinely open. diff --git a/sample-sites/modern-philosophy/pages/meta-05-mind.md b/sample-sites/modern-philosophy/pages/meta-05-mind.md new file mode 100644 index 0000000..a7b8a09 --- /dev/null +++ b/sample-sites/modern-philosophy/pages/meta-05-mind.md @@ -0,0 +1,61 @@ +--- +title: Philosophy of Mind +sort: 140 +section-id: metaphysics +description: Dualism, functionalism, physicalism, qualia, and the hard problem of consciousness. +language: en +--- + +# Philosophy of Mind + +Philosophy of mind asks what the mind is, how mental states relate to physical states, and whether consciousness can be explained by the natural sciences. It is a meeting point of metaphysics, epistemology, cognitive science, and neuroscience โ€” and at its centre lies what David Chalmers called *the hard problem of consciousness*, the question of why there is subjective experience at all. + +## Substance Dualism + +Descartes' *substance dualism* holds that mind and body are distinct substances: the body is extended substance (res extensa), governed by mechanical laws; the mind is thinking substance (res cogitans), unextended and not subject to physical laws ^[Descartes, R., *Meditations*, AT VII:78-80; *The Passions of the Soul*, AT XI:330]. + +Substance dualism captures the intuition that mental life โ€” the experience of pain, the feeling of red, the taste of coffee โ€” is radically different in kind from the physical world. No description in purely physical terms seems to capture what it is like to be in pain. + +The central objection: *causal interaction*. If mind and body are distinct substances, how do they causally interact? How does my decision to raise my arm cause my arm to rise? Descartes' attempted answer โ€” via the pineal gland โ€” was never convincing. Occasionalism (Malebranche) and pre-established harmony (Leibniz) were developed as alternatives, both of which deny genuine causal interaction and invoke God. + +## Behaviourism and Its Failures + +*Logical behaviourism* โ€” associated with Ryle and early Wittgenstein โ€” held that mental concepts are analysable in terms of behavioural dispositions, not inner states ^[Ryle, G., *The Concept of Mind*, 1949]. To believe that it will rain is to be disposed to carry an umbrella, to seek shelter, and so on. There is no "ghost in the machine" โ€” mentality just is the complex of behavioural dispositions. + +Hilary Putnam argued that behaviourism fails because behavioural dispositions are mediated by other mental states ^[Putnam, H., "Brains and Behavior", 1963]. The pain-disposition to withdraw from stimuli requires the desire to avoid pain; the belief-disposition to seek shelter requires the desire to stay dry. No purely behavioural analysis of a mental state can avoid this regress. + +## Identity Theory + +*Identity theory* โ€” associated with Place and Smart โ€” held that mental states are identical to brain states: pain is a type of neural activity ^[Place, U.T., "Is Consciousness a Brain Process?", *British Journal of Psychology* 47, 1956; Smart, J.J.C., "Sensations and Brain Processes", *Philosophical Review* 68, 1959]. + +**Multiple realisability objection:** Putnam argued that mental states are *multiply realisable* โ€” they can be realised in many different physical substrates ^[Putnam, H., "Psychological Predicates", 1967]. If octopuses (with radically different nervous systems) can be in pain, pain cannot be identical to any specific neural state. This counts against *type* identity theory, though not *token* identity (this instance of pain is identical to this neural event). + +## Functionalism + +*Functionalism* โ€” Putnam's alternative โ€” holds that mental states are defined by their *functional role*: their causal relations to sensory inputs, behavioural outputs, and other mental states ^[Putnam, H., "The Nature of Mental States", 1967]. Pain is whatever state is typically caused by tissue damage, causes withdrawal behaviour and the desire to relieve it, and interacts with other states in characteristic ways. + +Functionalism accommodates multiple realisability: what makes something a pain is its functional role, regardless of whether it is implemented in neurons, silicon, or anything else. It is the dominant view in philosophy of mind and cognitive science. + +**Objections:** The *inverted qualia* argument: two people might have their functional roles entirely aligned while having inverted phenomenal experiences (what's red to you is green to me). Functionalism, which is defined by function, cannot distinguish them. The *absent qualia* argument: a system could have all the right functional relations while having no phenomenal experience at all โ€” a philosophical zombie ^[Block, N., "Troubles with Functionalism", *Minnesota Studies in the Philosophy of Science* 9, 1978; Chalmers, D., *The Conscious Mind*, 1996, ch.3]. + +## Physicalism and Its Varieties + +Contemporary philosophy of mind is broadly physicalist: mental states are physical states or at least entirely dependent on physical states. The question is *how* they are dependent. + +*Supervenience physicalism*: mental properties supervene on physical properties โ€” any two individuals physically identical are mentally identical ^[Kim, J., *Supervenience and Mind*, Cambridge UP, 1993]. + +*Non-reductive physicalism*: mental properties are real but not reducible to physical properties, even though they supervene on them. + +*Reductive physicalism*: mental properties can ultimately be explained in physical terms. + +The *causal exclusion argument* (Kim) poses a serious problem for non-reductive physicalism: if physical events have sufficient physical causes, and mental events are supposed to cause behaviour, then either mental events are physical events (reductivism) or mental events are causally redundant ^[Kim, J., *Mind in a Physical World*, 1998, ch.2-3]. + +## The Hard Problem + +Chalmers distinguished the *easy problems* of consciousness โ€” explaining cognitive access, attention, introspection, sleep/waking cycles (these problems are hard, but they admit of functional-explanatory solutions) โ€” from *the hard problem*: why is there subjective experience at all? ^[Chalmers, D., "Facing Up to the Problem of Consciousness", *Journal of Consciousness Studies* 2, 1995]. + +Even a complete physical and functional account of what the brain does would leave open why any of this processing is *experienced* โ€” why it feels like something to be a brain. The explanatory gap between physical descriptions and phenomenal experience seems irreducible. + +Responses range from *type-B physicalism* (the gap is a conceptual illusion, not a real explanatory gap) to *property dualism* (phenomenal properties are real, non-physical properties that supervene on physical ones) to *panpsychism* (consciousness is a fundamental feature of reality, found at all levels of physical organisation) ^[Goff, P., *Galileo's Error*, 2019]. + +The hard problem has not been solved. Whether it is solvable within a physicalist framework, or whether it requires revising our fundamental ontology, remains one of philosophy's most contested open questions. diff --git a/sample-sites/modern-philosophy/pages/meta-06-time.md b/sample-sites/modern-philosophy/pages/meta-06-time.md new file mode 100644 index 0000000..9c8d6ed --- /dev/null +++ b/sample-sites/modern-philosophy/pages/meta-06-time.md @@ -0,0 +1,61 @@ +--- +title: The Nature of Time +sort: 150 +section-id: metaphysics +description: A-series and B-series, presentism, eternalism, and the growing block theory of time. +language: en +--- + +# The Nature of Time + +Time is both utterly familiar and deeply puzzling. We live in it, measure it, experience its passage. Yet when we ask what time is, whether the past and future exist, whether time flows or merely seems to, we find ourselves quickly in some of philosophy's most difficult territory. + +## McTaggart's A-Series and B-Series + +J.M.E. McTaggart introduced the most influential framework for the philosophy of time in his 1908 paper "The Unreality of Time" ^[McTaggart, J.M.E., "The Unreality of Time", *Mind* 17, 1908, pp.457-474]. + +The *B-series* is the ordering of events as earlier, simultaneous, and later. Every event stands in a fixed B-relation to every other: the Battle of Hastings is earlier than the French Revolution; your birth is earlier than your reading this sentence. These relations are *permanent*: if A is earlier than B, it always has been and always will be. + +The *A-series* orders events as *past*, *present*, and *future*. Unlike B-relations, A-properties change: what is now future becomes present and then past. The event of your reading this sentence was future, is now present, and will be past. + +McTaggart argued that the A-series is essential to time โ€” without the genuine distinction between past, present, and future, there would be no temporal becoming, no flow of time, and time would not be genuinely real. But the A-series is contradictory: every event has all three properties (past, present, future), which are mutually exclusive. Attempts to resolve the contradiction by saying "the event is present *now*, past *at later times*, future *at earlier times*" invoke further temporal moments and generate a vicious infinite regress. McTaggart concluded that time is unreal. + +Most philosophers reject the conclusion but accept that McTaggart identified a genuine structural puzzle. + +## Presentism + +*Presentism* holds that only present entities exist ^[Crisp, T., "Presentism", in *Oxford Handbook of Metaphysics*, 2003]. The past is gone; the future is not yet here. "There were dinosaurs" is true, but dinosaurs do not now exist in any sense โ€” they existed and no longer do. + +Presentism is the view that most naturally fits everyday temporal experience. The past seems gone; the future seems open. + +**The problem of cross-temporal relations:** Many true claims seem to relate present entities to past ones: Caesar crossed the Rubicon *before* you were born. If Caesar does not now exist, what makes this claim true? Presentists have responded with *truthmakers* that exist presently โ€” facts about the present state of the world โ€” and with *temporal ersatzism* (abstract representations of past times). + +**Reconciling presentism with special relativity:** Relativity implies there is no absolute simultaneity โ€” different reference frames carve the four-dimensional spacetime differently. This is deeply problematic for presentism, which requires a distinguished present ^[Putnam, H., "Time and Physical Geometry", *Journal of Philosophy* 64, 1967]. + +## Eternalism (The Block Universe) + +*Eternalism* โ€” the view associated with Einstein's spacetime โ€” holds that past, present, and future entities all equally exist, merely at different temporal locations ^[Sider, T., *Four-Dimensionalism*, Oxford UP, 2001, ch.2]. The universe is a four-dimensional "block" in which all events co-exist; the distinction between past, present, and future is merely perspectival, like the distinction between here and there. + +Eternalism accommodates special relativity naturally: there is no privileged present, and the temporal order of events can be frame-relative. + +**The problem of temporal passage:** Eternalism seems to make temporal experience mysterious. If past and future are equally real, why does time seem to pass? Why do we experience becoming rather than merely co-existing with our past and future selves in a static four-dimensional block? + +Some eternalists accept this implication and argue that the *passage* of time is an illusion โ€” a product of our temporal perspective, not a feature of reality. Others have argued that passage can be accommodated within an eternalist framework through the *moving spotlight* theory. + +## The Growing Block Theory + +C.D. Broad proposed an intermediate view: the growing block ^[Broad, C.D., *Scientific Thought*, 1923, pp.66-68]. The past and present are real and fixed; the future does not yet exist. As time passes, new events come into existence and the block grows. This preserves the reality of temporal becoming without committing to a privileged present. + +**Objections:** If the growing block is correct, and past moments are real and fixed, how do we know we are in the present rather than in some past moment (which is, after all, equally real)? This generates a peculiar form of scepticism about our temporal location ^[Tooley, M., *Time, Tense, and Causation*, Oxford UP, 1997]. + +## The Direction of Time + +Physics, at the fundamental level, is largely time-symmetric: the laws of mechanics, electromagnetism, and quantum mechanics (with minor exceptions) work equally in both temporal directions. Yet our experience of time is strongly asymmetric: we remember the past and not the future; causes precede effects; entropy increases. + +Boltzmann argued that the thermodynamic arrow of time โ€” the increase of entropy โ€” explains the asymmetry ^[Boltzmann, L., *Lectures on Gas Theory*, 1896-98]. We are in a low-entropy region of a vastly larger, mostly high-entropy universe; the Second Law of Thermodynamics is a statistical fact about macroscopic systems, not a fundamental law. + +The *causal theory* of time holds that the direction of time is grounded in the direction of causation: the future is the direction in which causal processes flow. But if causation itself is time-asymmetric, this seems circular. + +**Temporal experience:** Husserl's phenomenology of time-consciousness distinguished *retention* (the immediate just-past), *primal impression* (the now), and *protention* (the immediate just-future) as three interlocking structures of temporal awareness ^[Husserl, E., *On the Phenomenology of the Consciousness of Internal Time*, 1928]. The experience of time as flowing may be constituted by this structure rather than being evidence of metaphysical flow. + +The nature of time remains one of the most contested and most fundamental questions in metaphysics, connecting to physics, the theory of causation, personal identity, and the phenomenology of experience. diff --git a/sample-sites/modern-philosophy/pages/preface.md b/sample-sites/modern-philosophy/pages/preface.md new file mode 100644 index 0000000..d8e6193 --- /dev/null +++ b/sample-sites/modern-philosophy/pages/preface.md @@ -0,0 +1,48 @@ +--- +title: Preface +sort: 100 +section-id: front-matter +description: Professor Okafor's preface โ€” why philosophy matters, how to use this book, and acknowledgements. +language: en +--- + +![Library and scholarly atmosphere](assets/images/hero.jpg) + +# Preface + +Philosophy begins where certainty ends. It is the sustained, rigorous attempt to think carefully about questions that do not yield to experiment, calculation, or common sense alone โ€” questions about the nature of knowledge, the structure of reality, the foundations of morality, and the meaning of a human life. These questions are not trivial or marginal. They are, in many respects, the questions that underlie everything else we do. The scientist who believes her experiments can yield knowledge presupposes an account of knowledge and justification. The judge who sentences a criminal presupposes an account of moral responsibility and desert. The citizen who argues for a just social arrangement presupposes an account of fairness, rights, and legitimate authority. + +Philosophy is the discipline that examines these presuppositions โ€” that takes them seriously enough to interrogate them rather than quietly relying on them. + +This book is an introduction to three of philosophy's central subdisciplines: epistemology (the theory of knowledge), metaphysics (the study of the fundamental nature of reality), and ethics (the systematic examination of morality). These three areas are not the whole of philosophy โ€” there is also philosophy of language, philosophy of science, political philosophy in its own right, philosophy of mathematics, philosophy of religion, and many others โ€” but they constitute what has traditionally been called the core of the discipline, and they are deeply interconnected. Our account of knowledge bears on our account of what exists. Our account of what exists bears on our account of moral facts. The connections run in every direction. + +## On This Book + +*Foundations of Modern Philosophy* is written for advanced undergraduates and first-year graduate students who have some acquaintance with philosophical questions but no prior systematic training. It assumes intellectual seriousness but not technical background. The goal is not to survey every debate in every area โ€” that would require a library, not a textbook โ€” but to provide rigorous introductions to the central issues, equip the reader with the conceptual vocabulary needed to engage with primary sources, and convey something of the excitement of philosophy as a living intellectual enterprise. + +Each chapter introduces a major area of inquiry, presents the central arguments with the care they deserve, and points toward the ongoing debates that the reader will encounter if they pursue the subject further. The book is designed to be read sequentially, but chapters can also be read independently: those with a specific interest in, say, philosophy of mind or free will can begin there and follow the cross-references. + +I have tried to write with clarity without sacrificing rigour, and with genuine intellectual engagement without pretending that the questions have been settled. They have not been settled. Some of them may not be settleable. This is not a reason for despair but for sustained attention. + +## Philosophy and Disagreement + +One thing that surprises new students of philosophy is how much disagreement persists among careful, intelligent, well-informed people. In mathematics, experts eventually converge. In philosophy, they often do not. This is sometimes taken as evidence that philosophy makes no progress, or that philosophical questions are somehow empty. + +I believe this is wrong, for two reasons. + +First, philosophical progress is real, even when consensus is absent. We understand the problems more precisely than we did. The conceptual terrain has been mapped. Certain paths have been shown to be dead ends. The space of viable positions has been narrowed, even if it has not collapsed to a single point. + +Second, sustained intelligent disagreement is not a failure โ€” it is what happens when the questions are genuinely hard and the standards of evidence genuinely exacting. The fact that Kant, Mill, and Aristotle disagree about the foundations of morality does not mean there are no good arguments in ethics. It means the questions are difficult enough that even great minds approach them from different directions and reach different conclusions. + +Students who enter philosophy looking for certainty will be disappointed. Students who enter looking for difficult questions asked well, with intellectual honesty and genuine rigour, will find that and more. + +## Acknowledgements + +This book has accumulated debts over many years of teaching. Students at the University of Lagos, Cambridge, and Princeton asked the questions that forced me to think more carefully; colleagues in seminars and conference rooms challenged positions I held too comfortably; many generations of undergraduates reminded me, by their confusion and their insight alike, what it is actually like to encounter these ideas for the first time. + +I am grateful to my research assistants, Amara Osei-Bonsu and Lars Eriksson, for careful reading of the manuscript. My editor, who has the good philosopher's gift of asking exactly the right question at exactly the wrong moment, improved the book considerably. + +My deepest debts are to the philosophers whose work is discussed in these pages. I have tried to present their arguments with fairness. Where I have failed, the failure is mine. + +*James Okafor* +*Lagos / Princeton, 2026* diff --git a/sample-sites/modern-philosophy/pages/synthesis.md b/sample-sites/modern-philosophy/pages/synthesis.md new file mode 100644 index 0000000..478e428 --- /dev/null +++ b/sample-sites/modern-philosophy/pages/synthesis.md @@ -0,0 +1,77 @@ +--- +title: Synthesis and Open Questions +sort: 100 +section-id: conclusion +description: How epistemology, metaphysics, and ethics connect, and ten open problems that define the frontiers of contemporary philosophy. +language: en +--- + +# Synthesis and Open Questions + +We have traversed three great branches of philosophy โ€” epistemology, metaphysics, and ethics โ€” as if they were distinct territories. They are not. In this concluding chapter we trace some of the connections between them, then identify ten open problems that represent the active frontiers of contemporary philosophical inquiry. + +## How the Three Branches Connect + +### Epistemology and Metaphysics + +Epistemology and metaphysics are entangled at their foundations. The question "What is there?" (ontology) is inseparable from "How can we know what there is?" (epistemology). Kant made this explicit: our knowledge of reality is always knowledge of reality as structured by our cognitive apparatus. The thing-in-itself โ€” the world independent of our categories โ€” is unknowable. + +The debate between realism and anti-realism in metaphysics has a direct epistemological dimension: scientific realists hold that our best theories give us genuine knowledge of the unobservable structure of reality; anti-realists (van Fraassen's constructive empiricism) hold that empirical adequacy, not truth, is the goal of science. + +Scepticism is both an epistemological and a metaphysical thesis: if we cannot rule out the brain-in-a-vat hypothesis, then our beliefs about the external world may be systematically false. Responding to scepticism requires both an epistemological account of what knowledge requires and a metaphysical account of what makes beliefs true. + +### Metaphysics and Ethics + +Free will and determinism (Chapter 9) is the clearest intersection of metaphysics and ethics. Moral responsibility โ€” the foundation of our entire ethical and legal practice โ€” presupposes that agents could have done otherwise. If determinism is true (and it may be), then whether this condition is satisfied becomes a metaphysical question with profound ethical and social consequences. + +Personal identity (Chapter 7) connects to ethics in multiple ways. Derek Parfit argued that the correct view of personal identity โ€” that what matters in survival is not identity per se but psychological continuity โ€” has far-reaching implications for distributive justice, self-interest, and our concern for future persons. If I am only loosely connected to my future self, do I have the same reasons for prudence? + +### Epistemology and Ethics + +Moral epistemology asks whether we can have knowledge of ethical truths, and if so, how. Empiricists about ethics hold that moral judgements are answerable to experience; rationalists hold that some moral truths are knowable a priori. The methodology of ethics โ€” intuitions as data, reflective equilibrium, thought experiments โ€” is itself a set of epistemological commitments. + +The is-ought gap (Hume) is an epistemological constraint on ethical argument: purely factual premises cannot entail normative conclusions. This shapes what counts as a valid argument in ethics. + +## Ten Open Problems in Philosophy + +### 1. The Hard Problem of Consciousness +Why is there subjective experience at all? Why does processing information feel like anything? David Chalmers's formulation of the hard problem remains without consensus resolution. Physicalist accounts explain the functional and structural properties of mind but seem to leave out the *what it is like*. + +### 2. The Nature of Mathematical Objects +Are mathematical objects abstract entities that exist independently of minds (Platonism), or are they mental constructs (constructivism), or merely useful fictions (fictionalism)? The unreasonable effectiveness of mathematics โ€” its uncanny applicability to physical reality โ€” demands explanation on any view. + +### 3. The Reference of Natural Kind Terms +Kripke and Putnam argued that natural kind terms ("water," "gold") refer rigidly to their physical essences, not to descriptive clusters. But what determines reference? The causal-historical picture has unresolved problems for highly theoretical kinds (fields, virtual particles) and social/biological categories. + +### 4. The Status of Modality +What grounds claims about necessity and possibility? Are possible worlds Lewisian concrete universes, abstract maximally consistent sets of propositions, or something else? The ontological costs of modal realism seem high; the alternatives face their own problems. + +### 5. Personal Identity Over Time +Despite extensive philosophical work, we lack a fully satisfactory account of what makes you the same person you were ten years ago โ€” and what this should matter for ethics. Parfit's reductionism remains controversial. + +### 6. The Foundations of Probability +The three major interpretations โ€” frequentist, Bayesian (subjective), and propensity โ€” each face serious objections. The role of probability in quantum mechanics compounds the difficulty: are quantum probabilities objective features of reality or epistemic representations of our uncertainty? + +### 7. The Demarcation Problem +What distinguishes science from non-science, or good science from pseudo-science? Popper's falsifiability criterion is widely acknowledged to be insufficient. String theory and cosmological multiverse theories generate empirical predictions only under contested conditions. Philosophy of science lacks an agreed criterion. + +### 8. Moral Realism and Evolutionary Debunking +Sharon Street's evolutionary debunking argument: if our moral faculties were shaped by natural selection for fitness rather than moral truth, then we have no reason to trust that our moral intuitions track mind-independent moral facts.^[Street, S. (2006). "A Darwinian Dilemma for Realist Theories of Value." *Philosophical Studies*, 127(1).] Moral realists must either deflect this argument or explain why adaptive pressure would align our intuitions with moral truth. + +### 9. The Epistemology of Disagreement +When two equally well-informed, well-reasoned individuals reach opposing conclusions, what should each do? The *conciliationist* view says each should move towards the other; the *steadfast* view says you can maintain your view if you have independent reasons. The answer has implications for political philosophy, scientific consensus, and religious belief. + +### 10. The Grounds of Normativity +Why does reason bind us? Kant's answer โ€” that rational nature is the source of all value โ€” faces both metaphysical challenges (what is rational nature?) and sceptical challenges (why should I care about what reason prescribes?). Korsgaard's attempt to ground normativity in reflective self-endorsement has been contested. This may be the most fundamental question in all of philosophy. + +## Conclusion + +Philosophy does not progress by solving problems and discarding them. The questions examined in this book โ€” about knowledge, reality, and value โ€” are perennial because they arise from the structure of human thought itself. What philosophy offers is not definitive answers but greater clarity about what the questions are, greater rigour in evaluating proposed answers, and greater sensitivity to the hidden assumptions that shape all inquiry. + +The student who completes this introduction will find these questions following them into every discipline they pursue โ€” into science, into law, into medicine, into politics, into ordinary life. That is not a failure of philosophy to resolve itself. It is philosophy doing exactly what it should. + +## Further Reading + +- Chalmers, D. (2023). *Reality+: Virtual Worlds and the Problems of Philosophy*. Penguin. +- Parfit, D. (1984). *Reasons and Persons*. Oxford University Press. +- Nagel, T. (1986). *The View from Nowhere*. Oxford University Press. diff --git a/sample-sites/modern-philosophy/search.json b/sample-sites/modern-philosophy/search.json new file mode 100644 index 0000000..5b381b6 --- /dev/null +++ b/sample-sites/modern-philosophy/search.json @@ -0,0 +1,266 @@ +[ + { + "file": "pages/ep-01-knowledge.md", + "title": "What is Knowledge?", + "section-id": "epistemology", + "keywords": "", + "description": "The JTB analysis of knowledge, the Gettier problem, and the major responses to Gettier.", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# What is Knowledge?\n\nEpistemology โ€” from the Greek *episteme* (knowledge) and *logos* (account or study) โ€” is the branch of philosophy concerned with the nature, sources, and limits of knowledge. Its central question is deceptively simple: what is it to know something?\n\n## The Traditional Analysis: Justified True Belief\n\nThe dominant account in Western philosophy from Plato through most of the twentieth century held that knowledge is *justified true belief* (hereafter JTB). To know that *p* is to (1) believe that *p*, (2) have *p* be true, and (3) be justified in believing that *p*.\n\nEach condition plays a role. The truth condition rules out lucky coincidences: if I believe the train departs at 10am and it in fact departs at 10am, I do not know this if I formed the belief by guessing. The belief condition rules out propositions I accept without endorsing: I may act as if London is south of Edinburgh (it is not) without believing this, in which case I cannot be said to know it. The justification condition โ€” the most philosophically contested of the three โ€” distinguishes knowledge from mere true belief: if I believe, on no grounds whatsoever, that there is a spider behind the bookcase, and there happens to be a spider there, I do not thereby *know* there is a spider. Knowledge requires that one's belief be appropriately supported.\n\nThe JTB analysis has Platonic roots: in the *Meno*, Socrates distinguishes knowledge (*episteme*) from right opinion (*ortho doxa*) by the presence of an \"account\" that tethers the belief to its object. In the *Theaetetus*, Plato examines and ultimately rejects several definitions of knowledge, leaving the question famously open.\n\n## The Gettier Problem\n\nIn a short, devastating paper published in 1963, Edmund Gettier showed that the JTB analysis is insufficient ^[Gettier, E., \"Is Justified True Belief Knowledge?\", *Analysis* 23, 1963, pp.121-123]. He produced two counterexamples โ€” cases where an agent has a justified true belief but, intuitively, does not know.\n\nThe original cases are somewhat technical, but their structure can be illustrated as follows. Smith has good evidence that Jones will get the job, and that Jones has ten coins in his pocket. He infers: \"The man who will get the job has ten coins in his pocket.\" In fact, Smith himself gets the job โ€” and unbeknownst to him, he also has ten coins in his pocket. Smith's belief is true and justified. But he does not know it, because his justification for the belief is the evidence about Jones, not about himself. The truth of his belief is, in the relevant sense, a matter of luck.\n\nGettier cases share a common structure: the belief is true, and it is justified, but the justification and the truth are *accidentally* connected in a way that undermines knowledge. The epistemic luck that disqualifies knowledge is sometimes called *veritic luck* โ€” the belief could easily have been false, even given the justification.\n\n## Responses to Gettier\n\nThe philosophical literature on Gettier is vast ^[For surveys, see Shope, R., *The Analysis of Knowing*, Princeton UP, 1983; Ichikawa, J. and Steup, M., \"The Analysis of Knowledge\", *Stanford Encyclopedia of Philosophy*, 2018]. Several broad strategies have been pursued.\n\n**The No-False-Lemmas Approach.** Some responses add a fourth condition: knowledge requires that the justification not pass through any false beliefs. In the Smith-Jones case, Smith's inference passes through the false belief that Jones will get the job. This approach handles many Gettier cases but fails against variants that generate knowledge through no false belief.\n\n**Defeasibility Theories.** Lehrer and Paxson proposed that knowledge requires that one's justification not be *defeatable* by true information ^[Lehrer, K. and Paxson, T., \"Knowledge: Undefeated Justified True Belief\", *Journal of Philosophy* 66, 1969, pp.225-237]. If there is some truth that, were the agent to learn it, would undermine her justification, she does not know. This captures the intuition that Gettier cases involve misleading justification, but the correct formulation of the defeasibility condition has proven elusive.\n\n**Reliabilism.** Alvin Goldman proposed replacing the traditional internalist justification condition with an externalist one: knowledge requires that the belief be produced by a *reliable belief-forming process* ^[Goldman, A., \"What is Justified Belief?\", in *Justification and Knowledge*, ed. Pappas, Reidel, 1979]. A reliable process is one that tends to produce true beliefs in the actual world. Perception, memory, and valid inference are typically reliable; guessing and wishful thinking are not. Reliabilism handles Gettier cases naturally: if a belief is produced by a reliable process, there is no epistemic luck.\n\n**Safety and Sensitivity Conditions.** Sosa and Nozick proposed modal conditions on knowledge. Nozick's *tracking theory* required that the belief \"tracks\" the truth: if *p* were false, the agent would not believe *p* (the sensitivity condition); and if *p* were true, the agent would believe *p* (the adherence condition) ^[Nozick, R., *Philosophical Explanations*, Harvard UP, 1981, ch.3]. Sosa's *safety* condition required that the agent could not easily have been wrong ^[Sosa, E., \"How to Defeat Opposition to Moore\", *Philosophical Perspectives* 13, 1999].\n\n**Knowledge First.** Timothy Williamson has argued that the traditional project of analysing knowledge in terms of more basic conditions is fundamentally misguided ^[Williamson, T., *Knowledge and Its Limits*, Oxford UP, 2000]. Knowledge, he argues, is a primitive mental state โ€” not reducible to belief plus conditions. Rather than asking what conditions must supplement belief to yield knowledge, we should take knowledge as the starting point and explain belief and justification in terms of it. The slogan is \"knowledge first.\"\n\n## Internalism and Externalism\n\nThe Gettier debate brought into focus a broader dispute about the nature of epistemic justification. *Internalists* hold that the factors that determine whether a belief is justified must be *accessible* to the agent โ€” available through reflection alone ^[Chisholm, R., *Theory of Knowledge*, Prentice-Hall, 1966]. On this view, two agents who are internally identical (same beliefs, same phenomenal states) must be equally justified, even if their environments differ dramatically.\n\n*Externalists* deny this. Goldman's reliabilism is paradigmatically externalist: whether a belief is produced by a reliable process is a fact about the external world, not something the agent can determine by reflection. An agent might have a perfectly reliable belief-forming process that she has no way of knowing is reliable.\n\nThe internalism/externalism debate intersects with questions about scepticism (discussed in Chapter 5). Externalism offers a natural reply to sceptical scenarios โ€” brain-in-a-vat believers may have reliably formed beliefs even in their abnormal environment โ€” but faces the challenge of explaining the felt force of sceptical intuitions, which seem to appeal precisely to considerations accessible by reflection.\n\n## Knowledge and Understanding\n\nA growing body of work distinguishes *knowledge that* (propositional knowledge) from *knowledge how* (ability knowledge) and from *understanding*. Ryle's distinction between knowing-that and knowing-how influentially challenged the assumption that all knowledge is propositional ^[Ryle, G., *The Concept of Mind*, Hutchinson, 1949]. Understanding โ€” grasping why something is the case, how the pieces fit together โ€” seems to go beyond a collection of propositional beliefs and is increasingly seen as a distinct epistemic achievement worthy of investigation in its own right.\n\nThe question \"What is knowledge?\" turns out, as Plato suspected, to be genuinely difficult. The Gettier problem demonstrated that the most natural answer โ€” justified true belief โ€” is insufficient, and the subsequent fifty years of philosophy have not produced a consensus on what must replace it. But the failure to find a reductive analysis does not mean we have learned nothing. We have learned precisely why the question is hard, and that is progress." + }, + { + "file": "pages/ep-02-perception.md", + "title": "Perception and Reality", + "section-id": "epistemology", + "keywords": "", + "description": "Direct realism, indirect realism, idealism, and phenomenalism โ€” the major theories of perception and its relation to reality.", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Perception and Reality\n\nPerception is our most immediate route to knowledge of the external world, and yet it is philosophically treacherous. We trust our senses โ€” and then we discover that sticks look bent in water, towers look small from a distance, and the table that appears brown under incandescent light appears subtly different under daylight. These illusions and variations prompt an epistemological crisis: if our senses can mislead us, how can we trust them? And if we cannot fully trust them, what can we know about the world?\n\n## The Argument from Illusion\n\nThe *argument from illusion* is a traditional challenge to naive perceptual realism. It proceeds roughly as follows ^[Ayer, A.J., *The Foundations of Empirical Knowledge*, Macmillan, 1940]:\n\n1. In cases of illusion, what we are directly aware of (the bent stick, the shrunken tower) is not identical to the physical object.\n2. Perceptual experience in veridical (non-illusory) cases is intrinsically similar to experience in illusory cases.\n3. Therefore, what we are directly aware of even in veridical cases is not the physical object itself, but some intermediate object โ€” a *sense datum*, a subjective representation, a mental image.\n\nIf this argument is correct, we never perceive the external world directly. We perceive representations of it, and must infer the world from those representations.\n\n## Direct Realism\n\n*Direct realism* (also called naรฏve realism or common-sense realism) holds that in ordinary perception, we are directly aware of the physical world. There are no intermediary mental objects standing between us and the things we perceive.\n\nContemporary direct realists reject the argument from illusion by contesting its first premise. When I see the bent stick, I am not aware of some private sense datum; I am aware of the stick itself, and my experience has the representational content that the stick is bent โ€” which is a false content, but this does not require a separate object ^[Martin, M.G.F., \"The Transparency of Experience\", *Mind and Language* 17, 2002, pp.376-425].\n\n**Disjunctivism** is a sophisticated variant of direct realism that draws a fundamental distinction between veridical experience and illusion/hallucination ^[McDowell, J., \"Criteria, Defeasibility, and Knowledge\", *Proceedings of the British Academy* 68, 1982]. On this view, there is no common factor between seeing a tree and hallucinating a tree. Veridical perception genuinely consists in being acquainted with the object; hallucination is a numerically distinct kind of event that merely mimics it. This dissolves the argument from illusion by denying that veridical and illusory experiences must have the same fundamental nature.\n\n## Indirect Realism\n\n*Indirect realism* (or representationalism) accepts that we never perceive the external world directly. Our direct objects of experience are mental representations โ€” sense data, *qualia*, or \"ideas\" in the empiricist terminology. These representations are caused by, and typically resemble, the physical objects that produce them.\n\nLocke is the canonical indirect realist in the early modern period ^[Locke, J., *An Essay Concerning Human Understanding*, 1689, Book II]. He distinguished *primary qualities* (extension, shape, motion, number) โ€” features of objects that genuinely resemble our ideas of them โ€” from *secondary qualities* (colour, taste, smell, temperature) โ€” features that our ideas do not resemble; they are simply the powers of objects to produce certain experiences in us.\n\nIndirect realism faces a significant epistemological challenge: if we only ever directly perceive our representations, how can we know that those representations accurately track the external world? Locke acknowledged this; Berkeley exploited it to devastating effect.\n\n## Berkeley's Idealism\n\nGeorge Berkeley argued that indirect realism collapses into idealism ^[Berkeley, G., *A Treatise Concerning the Principles of Human Knowledge*, 1710]. If we only directly perceive ideas, and ideas are inherently mental, then matter โ€” that supposed cause of ideas existing independently of all minds โ€” is a philosopher's fiction. *Esse est percipi*: to be is to be perceived.\n\nBerkeley was not denying the existence of the ordinary objects of experience โ€” tables, trees, other people. He was claiming that their existence consists in their being perceived, either by finite minds or, when unobserved by us, by the mind of God. This is idealism, but of a commonsensical variety: Berkeley insisted his view was closer to common sense than Locke's.\n\nThe main objection to Berkeley is the arbitrariness of experience. If physical objects are collections of ideas, why do we not simply experience whatever we imagine? Berkeley's answer โ€” the regularity of experience is guaranteed by God โ€” is metaphysically expensive and not universally persuasive.\n\n## Phenomenalism\n\n*Phenomenalism* is a non-theistic descendant of Berkeley, associated with Hume, Mill, and twentieth-century logical empiricists like A.J. Ayer. Rather than reducing physical objects to ideas in God's mind, phenomenalism analyses statements about physical objects as equivalent to conditionals about what experiences would occur under certain conditions ^[Mill, J.S., *An Examination of Sir William Hamilton's Philosophy*, 1865; Ayer, A.J., *Language, Truth and Logic*, Gollancz, 1936].\n\n\"There is a table in the next room\" is analysed as something like: \"If anyone were to look in the next room under normal conditions, they would have table-experiences.\" The table is, in Mill's phrase, a \"permanent possibility of sensation.\"\n\nPhenomenalism faces serious difficulties with conditionals involving unfulfillable antecedents and with the enormous complexity required to capture ordinary physical-object claims in purely phenomenal terms. It has largely been abandoned as a research programme.\n\n## Contemporary Debates\n\nCurrent philosophy of perception engages with cognitive science and debates about the *format* of perceptual representation (is it propositional? imagistic? iconic?), the *reach* of perception (does it extend to abstract objects, high-level properties, or is it limited to low-level sensory features?), and the relationship between perception and belief ^[Siegel, S., *The Richness of the Senses*, Oxford UP, 2010].\n\nThe *enactivist* tradition, drawing on Merleau-Ponty's phenomenology, challenges representationalism from a different direction: perception, on this view, is not a matter of constructing internal representations but of active engagement with the environment ^[Noรซ, A., *Action in Perception*, MIT Press, 2004].\n\nThe debate between direct and indirect realism remains active and unresolved. What is clear is that perception โ€” however it ultimately works โ€” does not give us a transparent window onto the world; it gives us something whose relationship to the world requires careful philosophical examination." + }, + { + "file": "pages/ep-03-reason.md", + "title": "Reason and Rationalism", + "section-id": "epistemology", + "keywords": "", + "description": "Descartes, Leibniz, and Kant โ€” the rationalist tradition, a priori knowledge, and the role of reason in epistemology.", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Reason and Rationalism\n\n*Rationalism* is the view that reason โ€” independent of or prior to sensory experience โ€” is a significant source of knowledge. The paradigmatic rationalist claim is that some truths can be known *a priori*: known on the basis of reason alone, without appeal to experience. Mathematics and logic are the clearest cases. That 7 + 5 = 12, or that if all humans are mortal and Socrates is human then Socrates is mortal โ€” these seem knowable by pure thought, without conducting experiments or making observations.\n\n## The A Priori / A Posteriori Distinction\n\nThe distinction between *a priori* and *a posteriori* knowledge โ€” between knowledge independent of experience and knowledge dependent on it โ€” was systematised by Kant, though it has roots in earlier philosophy ^[Kant, I., *Critique of Pure Reason*, 1781/1787, B1-B6].\n\n*A priori* knowledge is justified independently of experience. It includes logical truths, mathematical truths, and perhaps certain conceptual truths (a bachelor is unmarried). Crucially, a priori knowledge is typically characterised by *necessity* and *universality*: a priori propositions are true in all possible worlds and admit of no exceptions.\n\n*A posteriori* (or *empirical*) knowledge is justified by experience. Contingent facts about the world โ€” there are seven continents, water is Hโ‚‚O, the temperature today is 22ยฐC โ€” are known a posteriori. Such propositions could have been otherwise, and we know them by observing the world.\n\nKant also introduced the analytic/synthetic distinction. An *analytic* judgment is one where the predicate is contained in the concept of the subject (\"All bachelors are unmarried\"). A *synthetic* judgment adds something beyond the subject concept (\"The cat is on the mat\"). Rationalists typically claim there is a priori synthetic knowledge โ€” knowledge that is both independent of experience and genuinely informative about the world. Kant thought mathematics and the principles of pure science were synthetic a priori.\n\n## Descartes and the Method of Doubt\n\nRenรฉ Descartes is the foundational figure of early modern rationalism. His *Meditations on First Philosophy* (1641) begins with systematic doubt: he resolves to suspend belief in anything he can doubt, to find, if anything survives, a foundation for knowledge that is absolutely certain ^[Descartes, R., *Meditations on First Philosophy*, AT VII:17-18].\n\nThe senses can deceive. Dreams can be indistinguishable from waking life. And most radically: could there be an evil demon, infinitely powerful and infinitely cunning, whose sole purpose is to deceive him? Under this hypothesis, even the truths of mathematics might be false.\n\nFrom this radical doubt, Descartes extracts one certain truth: *cogito ergo sum* โ€” \"I think, therefore I am.\" ^[Descartes, R., *Discourse on the Method*, AT VI:32]. Even if a demon deceives me, the deceiving requires that I exist as a thinking thing. The *cogito* survives the most radical doubt.\n\nFrom this single certainty, Descartes attempts to rebuild knowledge. He argues for the existence of a benevolent God who would not systematically deceive him, thereby reinstating trust in clear and distinct perception. The circularity of this reconstruction โ€” using clear and distinct perception to prove God's existence, then using God's existence to validate clear and distinct perception โ€” has been widely noted and is known as the *Cartesian circle* ^[Arnauld, A., *Fourth Objections*, in Descartes, *Meditations*, AT VII:214].\n\nDespite these difficulties, Descartes' contribution is foundational: he established the *epistemological turn* โ€” the idea that a systematic theory of knowledge is the prerequisite for metaphysics and science.\n\n## Leibniz: Necessary Truths and Monads\n\nGottfried Wilhelm Leibniz distinguished *truths of reason* (necessary truths, knowable a priori, the opposite of which is impossible) from *truths of fact* (contingent truths, known a posteriori, the opposite of which is conceivable) ^[Leibniz, G.W., *Monadology*, ยง33-34, 1714].\n\nFor Leibniz, the basic furniture of reality consists of *monads* โ€” immaterial, indivisible, mind-like substances. Each monad perceives (in a broad sense) every other monad, though with varying degrees of clarity. The apparent causal interaction between things is, in reality, a *pre-established harmony* installed by God: things do not genuinely cause each other but are programmed to correspond.\n\nLeibniz's principle of *sufficient reason* โ€” there must be a sufficient reason for everything being as it is rather than otherwise โ€” is a cornerstone of his system and has remained influential in metaphysics and the philosophy of science.\n\n## Kant's Copernican Revolution\n\nImmanuel Kant transformed the rationalism/empiricism debate with his *Critique of Pure Reason* (1781). He accepted from the rationalists that there is genuine a priori knowledge and from the empiricists that all knowledge *begins* with experience. His synthesis: experience is possible only because the mind structures it using a priori *forms* (space and time) and *categories* (substance, causation, necessity, and others).\n\nKant called this the *Copernican revolution* in philosophy ^[Kant, I., *Critique of Pure Reason*, Bxvi]: just as Copernicus moved the sun to the centre, Kant moved the knowing subject. We do not passively receive an already-structured world; we actively structure the world we experience, using the forms of intuition and the categories of the understanding.\n\nThis generates *transcendental idealism*: objects as we know them (*phenomena*) are partly constituted by our cognitive apparatus. Things as they are in themselves (*noumena*) โ€” beyond the conditions of our experience โ€” are unknowable.\n\nThe great achievement of Kant's epistemology is explaining how synthetic a priori knowledge is possible: mathematical and scientific principles are synthetic a priori because they describe the structure that the mind imposes on experience, not features of mind-independent reality. The cost is that our knowledge is bounded by the limits of possible experience.\n\n## The Analytic Critique\n\nThe logical empiricists of the early twentieth century (Carnap, Schlick, Ayer) challenged the very possibility of synthetic a priori knowledge ^[Ayer, A.J., *Language, Truth and Logic*, ch.4]. On their view, apparent a priori knowledge either reduces to analytic truths (true by definition) or is meaningless. Mathematics is analytic โ€” true by virtue of the meanings of mathematical terms. There are no synthetic a priori truths.\n\nQuine's \"Two Dogmas of Empiricism\" (1951) challenged even the analytic/synthetic distinction itself, arguing that no proposition is immune from revision in light of experience ^[Quine, W.V.O., \"Two Dogmas of Empiricism\", *Philosophical Review* 60, 1951]. This radical empiricism has been broadly influential but is not without critics ^[Grice, P. and Strawson, P., \"In Defense of a Dogma\", *Philosophical Review* 65, 1956].\n\nThe status of a priori knowledge remains one of epistemology's central contested questions." + }, + { + "file": "pages/ep-04-empiricism.md", + "title": "Empiricism", + "section-id": "epistemology", + "keywords": "", + "description": "Locke, Berkeley, and Hume โ€” the empiricist tradition and the limits of sensory knowledge.", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Empiricism\n\n*Empiricism* is the epistemological view that knowledge derives from, and must be grounded in, sensory experience. Where rationalism privileges reason, empiricism insists that the mind at birth is a *tabula rasa* โ€” a blank slate โ€” and that all our concepts and knowledge are acquired through experience. The great British empiricists of the seventeenth and eighteenth centuries โ€” John Locke, George Berkeley, and David Hume โ€” explored this view with increasing rigour and found, perhaps to their own surprise, that it leads to deeply uncomfortable places.\n\n## Locke's Empiricism\n\nJohn Locke set the agenda for British empiricism in his *Essay Concerning Human Understanding* (1689). His starting point is a polemic against innate ideas: there are no ideas or principles inscribed in the mind from birth, contrary to what Descartes and Leibniz held ^[Locke, J., *Essay*, I.ii].\n\nAll ideas originate in experience, which Locke divides into two kinds: *sensation* (the senses providing ideas of external objects) and *reflection* (the mind observing its own operations โ€” thinking, doubting, willing, perceiving). From simple ideas given in experience, the mind constructs *complex ideas* by combination, abstraction, and relation.\n\nLocke's primary/secondary quality distinction (discussed in the previous chapter) is central to his epistemology. We have genuine knowledge only of relations among our own ideas; the extent to which our ideas correspond to mind-independent reality is limited to the primary qualities.\n\nLocke's *Essay* is a monument of systematic empiricism, but it contains significant tensions. His account of substance โ€” the \"I know not what\" that underlies the qualities we perceive โ€” sits uneasily with his empiricism, since no experience corresponds to substance itself.\n\n## Hume's Fork and Bundle Theory\n\nDavid Hume pushed empiricism to its systematic conclusions with greater rigour and less embarrassment about where they led. In the *Treatise of Human Nature* (1739-40) and the *Enquiry Concerning Human Understanding* (1748), he developed what has been called the most powerful case in the history of philosophy for the limits of reason and knowledge.\n\n*Hume's fork* divides all genuine claims to knowledge into two classes ^[Hume, D., *Enquiry*, ยง4]:\n\n1. **Relations of ideas** โ€” propositions knowable a priori by reason alone, whose denials are contradictions (mathematics, logic, conceptual truths). These are certain but tell us nothing about the actual world.\n\n2. **Matters of fact** โ€” propositions about the world, known a posteriori through experience. Their denials are conceivable. They are contingent and can only be known through experience.\n\nHume's criterion of empirical significance follows: any meaningful claim is either a relation of ideas or a matter of fact. Claims that fit neither category โ€” much of traditional metaphysics, theology, and rationalist philosophy โ€” are, famously, \"nothing but sophistry and illusion\" ^[Hume, D., *Enquiry*, ยง12.3].\n\nOn the self, Hume is radically deflationary. When he introspects, he finds no impression of a persistent, unified self โ€” only \"a bundle or collection of different perceptions, which succeed each other with inconceivable rapidity\" ^[Hume, D., *Treatise*, I.iv.6]. The self, on his view, is a fiction constructed from the successive flow of impressions and ideas. Personal identity is a matter of psychological continuity, not a metaphysical substance.\n\n## The Problem of Induction\n\nHume's most influential contribution to epistemology is his analysis of inductive inference. We routinely infer, from past regularities, what will happen in the future: the sun has risen every morning, so it will rise tomorrow. All known emeralds have been green, so the next emerald will be green. What justifies these inferences?\n\nNot deductive reason: there is no logical contradiction in the sun's failing to rise. Not experience: to justify induction by appeal to the fact that induction has worked before is circular โ€” it assumes the very principle in question ^[Hume, D., *Treatise*, I.iii.6].\n\nHume's conclusion: our habit of inductive inference is a psychological necessity โ€” we cannot help forming expectations from regularities โ€” but it has no rational justification. This is the *problem of induction*, sometimes called \"Hume's guillotine\" for the way it cuts off a seemingly obvious route to empirical knowledge.\n\nThe problem of induction has proved enormously productive. Karl Popper's falsificationism โ€” the view that science proceeds by bold conjecture and attempted refutation rather than inductive generalisation โ€” was an explicit response ^[Popper, K., *The Logic of Scientific Discovery*, 1934]. Nelson Goodman's \"new riddle of induction\" showed that the problem was deeper than Hume recognised: even if induction is sometimes reliable, we need a further principle to determine which regularities to project onto the future ^[Goodman, N., *Fact, Fiction and Forecast*, 1955].\n\n## Hume on Causation\n\nHume's analysis of causation is equally influential. We believe that causes necessitate their effects โ€” that fire *must* produce heat, that one billiard ball *must* move another when struck. But examining our impressions, Hume finds no impression of *necessary connection* between events ^[Hume, D., *Enquiry*, ยง7]. We observe constant conjunction โ€” event A is always followed by event B โ€” and we observe the spatial and temporal contiguity of cause and effect. But necessity itself is never observed.\n\nHume's account: the idea of necessary connection is a projection of our own psychological tendency to expect B after A, given repeated experience of their conjunction. The \"necessary connection\" is in us, not in the world.\n\nThis has generated enormous controversy. The *regularity theory* of causation โ€” in Hume's footsteps โ€” holds that causation just is constant conjunction (plus contiguity and temporal priority). Counterfactual theories, mechanistic theories, and probabilistic theories have all been proposed as improvements.\n\n## Empiricism's Legacy\n\nEmpiricism as a systematic research programme continues in analytic philosophy. The logical empiricists (Carnap, Schlick, Neurath) developed a sophisticated version oriented toward the philosophy of science. Quine's naturalised epistemology โ€” the view that epistemology is continuous with empirical psychology โ€” is the most radical empiricist programme of the twentieth century ^[Quine, W.V.O., \"Epistemology Naturalized\", in *Ontological Relativity and Other Essays*, 1969].\n\nThe permanent contribution of the classical empiricists is methodological: the insistence that philosophical claims be answerable to experience, and the willingness to follow the logic of that insistence into uncomfortable territory." + }, + { + "file": "pages/ep-05-scepticism.md", + "title": "Scepticism and Its Responses", + "section-id": "epistemology", + "keywords": "", + "description": "Cartesian scepticism, the brain-in-a-vat scenario, contextualism, and relevant alternatives theories.", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Scepticism and Its Responses\n\nScepticism is the philosophical position that knowledge โ€” or at least, some significant domain of knowledge โ€” is impossible. It has been a central problem in epistemology from antiquity to the present, partly because it is remarkably difficult to refute and partly because attempting to refute it has driven some of philosophy's most creative work.\n\n## Ancient Scepticism\n\nThe ancient Greek sceptics โ€” Pyrrho of Elis, and later the Academic sceptics including Arcesilaus and Carneades โ€” argued that for any claim, there is an equally strong case for its denial, leaving the rational response one of *epochรฉ*: suspension of judgment ^[Sextus Empiricus, *Outlines of Pyrrhonism*, I.8-12, c.200 CE]. Suspension of judgment, they argued, brings *ataraxia* โ€” tranquillity, freedom from the anxiety that dogmatic belief produces.\n\nAncient scepticism was primarily a practical philosophy โ€” a way of living without commitment to metaphysical positions. Modern scepticism takes a different form: it is primarily an epistemological challenge, asking whether knowledge is possible given the limitations of our access to reality.\n\n## Cartesian Scepticism\n\nDescartes' sceptical scenarios, introduced in the *Meditations* as methodological tools, have become the canonical statements of modern epistemological scepticism. The *dreaming argument*: I cannot rule out that I am dreaming right now, and if I might be dreaming, I cannot be certain that anything I currently believe is true ^[Descartes, *Meditations*, AT VII:19].\n\nThe *evil demon hypothesis* is more radical: suppose there is an infinitely powerful deceiving demon who ensures that all my beliefs โ€” including the deliverances of reason and mathematics โ€” are false. I cannot disprove this. Therefore, I cannot be certain of anything.\n\nThe sceptical strategy exploits what has been called *epistemic closure*: if I know that P entails Q, and I know P, then I know Q. Equivalently: if I don't know Q, and P entails Q, then I don't know P ^[Nozick, R., *Philosophical Explanations*, p.204]. Sceptics argue: if I knew that I have hands, I would know that I am not a brain in a vat; I do not know that I am not a brain in a vat; therefore, I do not know that I have hands.\n\n## The Brain-in-a-Vat Scenario\n\nHilary Putnam updated Descartes' evil demon into the brain-in-a-vat scenario ^[Putnam, H., *Reason, Truth and History*, 1981, ch.1]. Suppose my brain has been removed from my body, placed in a vat, and is being fed electrical signals by a supercomputer that simulates a complete reality. All my experiences are exactly as they would be in normal embodied life.\n\nPutnam argued โ€” controversially โ€” that this scenario is incoherent. A brain in a vat lacks the causal connections to the external world that are necessary for its terms to refer to external objects. \"Water,\" as a brain-in-a-vat thinks it, does not refer to Hโ‚‚O โ€” it refers, at most, to the computer simulation. So a brain-in-a-vat thinking \"I am not a brain in a vat\" is producing a true sentence โ€” because its words do not refer to the things that would make it false. This semantic argument against scepticism has been widely discussed and contested.\n\n## Responses to Scepticism\n\n**Moorean Responses.** G.E. Moore's response to scepticism was blunt: we know more certainly that we have hands than we know any philosophical premise used in the argument for scepticism ^[Moore, G.E., \"Proof of an External World\", *Proceedings of the British Academy* 25, 1939]. The *modus ponens* can be run in either direction: from the premises to the sceptical conclusion, or from the falsity of the sceptical conclusion to the falsity of one of the premises. Moore insisted the latter is more reasonable.\n\nWittgenstein developed a related response: certain propositions โ€” \"There are physical objects,\" \"The world has existed for many years\" โ€” function as *hinges* that cannot be doubted within any practice of inquiry, because doubting them would not be coherent inquiry but something else entirely ^[Wittgenstein, L., *On Certainty*, ยง341, 1951].\n\n**Relevant Alternatives.** Fred Dretske proposed that knowledge requires ruling out only *relevant* alternatives โ€” possibilities that are live given one's actual situation ^[Dretske, F., \"Epistemic Operators\", *Journal of Philosophy* 67, 1970]. The possibility that I am a brain in a vat is not a relevant alternative in ordinary contexts; I do not need to rule it out to know that I have hands. Scepticism artificially expands the class of alternatives that must be eliminated.\n\n**Contextualism.** David Lewis and Stewart Cohen developed contextualist responses: the standards for knowledge vary with context ^[Lewis, D., \"Elusive Knowledge\", *Australasian Journal of Philosophy* 74, 1996; Cohen, S., \"How to be a Fallibilist\", *Philosophical Perspectives* 2, 1988]. In ordinary contexts, we correctly say we know many things. In sceptical philosophical discussions, where very high standards are in play, those knowledge attributions are false โ€” but this does not undermine ordinary attributions, which operate at a lower standard. Scepticism is a local phenomenon of artificially elevated epistemic standards.\n\n**Externalist Responses.** If knowledge requires reliably produced beliefs (as reliabilism holds), then the sceptical demon scenario involves beliefs that are not reliably produced and therefore do not constitute knowledge. But Descartes' scenario is just that โ€” a scenario where knowledge fails. This does not show that we actually lack knowledge in the real world.\n\n## Closure Denial\n\nNozick's tracking theory denied epistemic closure, which blocks the sceptical argument at its source ^[Nozick, R., *Philosophical Explanations*, pp.204-211]. On his account, I know I have hands because if I didn't have hands I wouldn't believe I did (the sensitivity condition). But I do not know I am not a brain in a vat โ€” because if I were a brain in a vat, I would still believe I wasn't (the sensitivity condition fails). Yet the failure to know the second proposition does not undermine knowledge of the first, because the inference from \"I have hands\" to \"I am not a brain in a vat\" is not knowledge-preserving on Nozick's account.\n\nThis is elegant, but the denial of closure is philosophically costly and has not won widespread acceptance.\n\n## The Significance of Scepticism\n\nScepticism matters not primarily because it is a live hypothesis that reflective people adopt, but because engaging with it illuminates the structure of our knowledge and the character of epistemic justification. The sceptical challenge to close our eyes and demonstrate that we know anything about the external world has driven epistemologists to produce their most careful accounts of justification, reliability, and the conditions for knowledge. Scepticism is philosophy's sharpening stone." + }, + { + "file": "pages/ep-06-truth.md", + "title": "Theories of Truth", + "section-id": "epistemology", + "keywords": "", + "description": "Correspondence, coherence, pragmatist, and deflationary theories of truth.", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Theories of Truth\n\nWhat is truth? The question seems either trivially easy or impossibly hard. Everyone knows that to say something true is to say how things are. And yet when we ask what this \"correspondence\" between language and world consists in, or whether there might be truths that do not correspond to any independent reality, we find ourselves in deep philosophical waters.\n\n## The Correspondence Theory\n\nThe *correspondence theory of truth* is the classical answer: a proposition (or belief, or statement) is true if and only if it corresponds to the facts ^[Aristotle, *Metaphysics*, 1011b26-28]. To say that snow is white is to say something true, because it corresponds to the fact that snow is white.\n\nThe appeal of this theory is its fidelity to common sense. When we assert something, we are attempting to describe how things are, and truth is the property of succeeding in that attempt.\n\nThe challenge is making \"correspondence\" precise. Early twentieth-century philosophy developed the *picture theory of meaning* (Wittgenstein's *Tractatus*, Russell's logical atomism) on which propositions picture facts by sharing their logical structure ^[Wittgenstein, L., *Tractatus Logico-Philosophicus*, 1921, ยง2.15; Russell, B., \"The Philosophy of Logical Atomism\", *Monist* 28, 1918]. This was technically ambitious but ultimately abandoned.\n\nThe central difficulty for correspondence theories is the nature of *facts*. Are facts mind-independent features of the world? If so, what exactly are they, and how do they differ from merely true propositions? The proliferation of suspect entities (negative facts, disjunctive facts, mathematical facts) has made many philosophers wary.\n\n## The Coherence Theory\n\nThe *coherence theory* identifies truth with coherence โ€” membership in a system of beliefs that are mutually consistent, mutually supporting, and comprehensive ^[Bradley, F.H., *Essays on Truth and Reality*, 1914]. On this view, a belief is true not because it corresponds to some external fact but because it coheres with the overall system of beliefs.\n\nThe coherence theory is associated with British idealism (Bradley, Bosanquet) and has been influential in anti-realist philosophy more generally. Its appeal lies in removing the mysterious \"correspondence\" relation: truth is an internal relation among beliefs, not a relation between thought and world.\n\nCritics raise two main objections. First, coherence is not sufficient for truth: many consistent, mutually supporting sets of beliefs are simply false. The elaborate beliefs of a deeply mistaken scientific tradition may be perfectly coherent. Second, coherence is not necessary: isolated beliefs can be true without being embedded in a rich coherent system.\n\n## Pragmatist Theories\n\nWilliam James and John Dewey developed *pragmatist* theories of truth that identified truth with what \"works\" โ€” what is expedient to believe, what guides successful action ^[James, W., \"Pragmatism's Conception of Truth\", in *Pragmatism*, 1907; Dewey, J., *Logic: The Theory of Inquiry*, 1938].\n\nJames's formulation is the most quotable: \"True ideas are those that we can assimilate, validate, corroborate and verify. False ideas are those that we cannot.\"\n\nPragmatism has been persistently misread as claiming that truth is whatever we find convenient to believe, or that powerful people's beliefs are therefore true. More charitably, pragmatism is the claim that our concept of truth is tied to the role beliefs play in guiding inquiry and action: there is no coherent notion of truth that is entirely divorced from human practice.\n\nThe strongest objection is that some truths are inaccessible to human inquiry โ€” truths about the distant past, truths about microscopic phenomena not yet observed. If truth is tied to what we could in principle verify, we seem to be claiming that there are no truths about such matters, which is implausible.\n\nPeirce's more sophisticated version identified truth with what ideal inquiry would converge on in the long run ^[Peirce, C.S., \"How to Make Our Ideas Clear\", *Popular Science Monthly* 12, 1878]. This avoids the problem of currently inaccessible truths but introduces a counter-factual notion of \"ideal inquiry\" that is difficult to cash out.\n\n## The Deflationary Theories\n\n*Deflationary theories* โ€” including Ramsey's *redundancy theory*, Quine's *disquotational theory*, and Horwich's *minimalism* โ€” hold that \"is true\" adds nothing to a proposition ^[Ramsey, F.P., \"Facts and Propositions\", *Proceedings of the Aristotelian Society* Supp. Vol. 7, 1927; Horwich, P., *Truth*, Blackwell, 1990].\n\nTo say \"it is true that snow is white\" is simply to say \"snow is white.\" The predicate \"is true\" serves a logical function โ€” enabling us to endorse propositions without repeating them, to quantify over propositions (\"everything she said is true\") โ€” but there is no deep property of *truth* to be analysed.\n\nThe T-schema captures the deflationist insight: for any sentence S, \"'S' is true if and only if S.\" There is nothing more to truth than this biconditional schema.\n\nDeflationists must explain how the truth predicate can do its logical work without denoting a substantive property. Critics also note that scientific realism seems to require a robust notion of truth โ€” the success of science is best explained by the approximate truth of scientific theories โ€” and deflationists have struggled to accommodate this.\n\n## Truth and Language\n\nThe relationship between truth and language raises further questions. Alfred Tarski's semantic theory of truth ^[Tarski, A., \"The Concept of Truth in Formalized Languages\", 1935] provided a technically rigorous account for formal languages: \"Snow is white\" is true in language L if and only if snow is white. For natural languages, which are semantically open, Tarski's approach encounters the *Liar paradox* (\"This sentence is false\") and related self-referential difficulties.\n\nContemporary philosophy of language has explored truth-conditional semantics (meaning just is truth conditions), pluralism about truth (different truth predicates for different domains โ€” factual, moral, mathematical), and relativism (truth relative to standards or contexts).\n\nThe debate about truth intersects with debates about realism and anti-realism, metaphysics, and the philosophy of language. It is one of the crossroads of philosophy โ€” a place where multiple independent routes converge." + }, + { + "file": "pages/eth-01-foundations.md", + "title": "Foundations of Ethics", + "section-id": "ethics", + "keywords": "", + "description": "Metaethics versus normative ethics, the question of moral realism, and why ethical theory matters for practical reasoning.", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Foundations of Ethics\n\nEthics is the branch of philosophy concerned with questions about value, obligation, and the good life. Before we can adjudicate between competing moral theories โ€” utilitarian, Kantian, Aristotelian โ€” we must first examine what kind of inquiry ethics is. This is the domain of *metaethics*.\n\n## The Metaethical Questions\n\nMetaethics asks: What is the nature of moral claims? When we say \"Torturing innocents is wrong,\" are we:\n\n1. Stating an objective fact about the world?\n2. Expressing a subjective attitude?\n3. Issuing a kind of command or prescription?\n4. Doing something altogether different?\n\nThese are not merely academic quibbles. The answer constrains what normative ethics can hope to achieve. If there are no moral facts, then ethical argument collapses into persuasion. If there are moral facts but we cannot know them, then ethical confidence is always epistemically precarious.\n\n## Moral Realism\n\nMoral realists hold that there are objective moral facts โ€” facts that hold independently of what any individual or culture believes. G.E. Moore (1903) argued that \"good\" is a simple, indefinable, non-natural property that we perceive through a kind of moral intuition.^[Moore, G.E. (1903). *Principia Ethica*. Cambridge University Press.]\n\nNaturalistic moral realists, by contrast, identify moral properties with natural properties. Cornell realists such as Peter Railton and Richard Boyd argue that moral terms refer to natural facts about human flourishing, desire-satisfaction, or social coordination.^[Boyd, R. (1988). \"How to be a Moral Realist.\" In Sayre-McCord (ed.), *Essays on Moral Realism*.]\n\nThe **open question argument** (Moore) challenges naturalism: for any natural property N, it is always an open question whether something that is N is thereby good. If \"good\" just meant \"maximises pleasure,\" then \"Is pleasure-maximising action good?\" would be a tautology โ€” but it is not. This suggests that moral properties are not identical to natural ones.\n\n## Anti-Realist Positions\n\n### Expressivism\nA.J. Ayer's *Language, Truth and Logic* (1936) presented the classic emotivist thesis: moral statements are not truth-apt at all. \"Stealing is wrong\" means something like \"Boo, stealing!\" โ€” it expresses disapproval rather than describing a fact.^[Ayer, A.J. (1936). *Language, Truth and Logic*. Gollancz.]\n\nSimon Blackburn developed *quasi-realism* to address the main objection to expressivism: that moral statements appear in contexts (conditionals, embedded clauses) where purely expressive readings are implausible. \"If stealing is wrong, then getting your brother to steal for you is also wrong\" cannot mean \"If boo stealing, then boo getting your brother to steal.\"\n\n### Error Theory\nJ.L. Mackie (1977) accepted that moral statements purport to state facts but argued they are systematically false. There are no objective moral properties in the world. We are all making a kind of category error when we assert moral claims.^[Mackie, J.L. (1977). *Ethics: Inventing Right and Wrong*. Penguin.] Mackie's *argument from queerness* claims that objective moral properties would be entities of a very strange kind โ€” utterly unlike anything in the natural world โ€” and our capacity to know them would require an equally strange epistemic faculty.\n\n### Constructivism\nKantian constructivists (Christine Korsgaard, John Rawls) occupy a middle position: moral truths are not mind-independent facts discovered by intuition, but neither are they merely expressions of attitude. They are constructed through procedures of rational reflection or agreement under idealised conditions. Moral facts are *the output* of a normative procedure, not independently existing objects.^[Rawls, J. (1980). \"Kantian Constructivism in Moral Theory.\" *Journal of Philosophy*, 77(9).]\n\n## Normative Ethics: An Overview\n\nNormative ethics asks: what ought we to do, and why? Three traditions dominate:\n\n| Tradition | Central Question | Key Figure |\n|---|---|---|\n| Consequentialism | What outcomes should we produce? | John Stuart Mill |\n| Deontology | What duties bind us regardless of outcome? | Immanuel Kant |\n| Virtue Ethics | What kind of person should I be? | Aristotle |\n\nEach tradition is examined in subsequent chapters. Here we note that they often converge in practice while diverging in their theoretical foundations โ€” a useful starting heuristic.\n\n## Moral Epistemology\n\nHow do we come to know moral truths (assuming there are any)? Candidates include:\n\n**Moral intuition** โ€” Direct, non-inferential moral knowledge. Strong intuitions (that gratuitous cruelty is wrong) are treated as data points that any adequate theory must accommodate. The method of *reflective equilibrium* (Rawls) involves moving back and forth between intuitions and principles until they cohere.\n\n**Moral perception** โ€” On some realist accounts, we literally perceive moral properties as we perceive colours (though with a different faculty). This view faces difficulty explaining inter-subjective disagreement.\n\n**Reason alone** โ€” Kantians hold that moral knowledge is a priori, derived from pure practical reason. We shall examine this in detail in the chapter on deontology.\n\n## The Fact-Value Distinction\n\nHume's famous observation โ€” that we cannot derive an \"ought\" from an \"is\" โ€” remains one of the most contested claims in metaethics.^[Hume, D. (1740). *A Treatise of Human Nature*, III.i.1.] If no purely factual description of the world entails a moral conclusion, then moral premises are always smuggled into ethical arguments. Naturalists must either deny the is-ought gap or explain why the gap does not undermine their position.\n\n## Relativism and Universalism\n\n*Cultural moral relativism* โ€” the descriptive claim that moral codes vary across cultures โ€” is well-documented. *Moral relativism* โ€” the normative claim that what is right depends on cultural norms โ€” is a separate and far more contested thesis. It generates self-refutation problems: if morality is relative, then the moral principle \"we should not impose our moral views on other cultures\" is itself only relatively binding.\n\nUniversalists hold that certain moral truths โ€” concerning dignity, suffering, basic rights โ€” apply to all humans in all contexts. The debate between particularism and universalism remains unresolved.\n\n## Further Reading\n\n- Parfit, D. (2011). *On What Matters*, Vols. Iโ€“II. Oxford University Press.\n- Schroeder, M. (2010). *Noncognitivism in Ethics*. Routledge.\n- Sayre-McCord, G. (ed.) (1988). *Essays on Moral Realism*. Cornell University Press." + }, + { + "file": "pages/eth-02-consequentialism.md", + "title": "Consequentialism", + "section-id": "ethics", + "keywords": "", + "description": "Utilitarian and consequentialist ethics from Bentham and Mill to Peter Singer and contemporary debates about act versus rule consequentialism.", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Consequentialism\n\nConsequentialism is the family of ethical theories holding that the moral quality of an action is entirely determined by its consequences. The right action is whichever action produces the best outcome. This apparently simple thesis generates a remarkably rich โ€” and contested โ€” philosophical programme.\n\n## Bentham's Utilitarianism\n\nJeremy Bentham (1748โ€“1832) founded classical utilitarianism on the *principle of utility*: actions are right insofar as they promote happiness, and wrong insofar as they promote unhappiness. By \"happiness,\" Bentham meant pleasure and the absence of pain.^[Bentham, J. (1789). *Introduction to the Principles of Morals and Legislation*. Payne.]\n\nBentham proposed the *felicific calculus* โ€” a method for quantifying pleasure and pain along seven dimensions: intensity, duration, certainty, propinquity, fecundity, purity, and extent. The theory is rigorously impartialist: \"each to count for one and none for more than one.\" The pleasure of a street cleaner counts exactly as much as that of an aristocrat.\n\n## Mill's Refinements\n\nJohn Stuart Mill (1806โ€“1873) accepted the utilitarian framework but argued that pleasures differ not only in quantity but in quality. *Higher pleasures* โ€” intellectual enjoyment, moral sentiment, aesthetic experience โ€” are intrinsically more valuable than lower, merely sensory pleasures.^[Mill, J.S. (1863). *Utilitarianism*. Parker, Son, and Bourn.] \"It is better to be Socrates dissatisfied than a fool satisfied.\"\n\nMill also attempted a consequentialist defence of rights and justice: rights protect interests so important that no ordinary gain in welfare could justify violating them. Whether this defence succeeds โ€” whether rights can be grounded in utility without collapsing into mere policy instruments โ€” remains debated.\n\n## Act and Rule Consequentialism\n\n**Act consequentialism** holds that each individual action should be evaluated by its consequences. The right act is the one that, among all available alternatives, produces the greatest aggregate welfare.\n\n**Rule consequentialism** holds that we should follow rules whose general adoption would produce the best consequences. We do not evaluate each act individually but ask: \"What rule, if generally followed, would produce the best outcomes?\" Rule consequentialism preserves more intuitive commitments about promise-keeping and justice: keeping a promise may not maximise utility on a particular occasion, but a rule requiring promise-keeping generally does.\n\nThe objection to act consequentialism is that it seems to justify intuitively monstrous acts whenever the mathematics works out. If torturing an innocent person would prevent a slightly larger number of harms, act consequentialism apparently demands it. The *separateness of persons* objection (Rawls) argues that consequentialism fails to respect the distinction between persons, treating them merely as vessels for welfare rather than as individuals with their own claims.\n\n## Peter Singer and Preference Utilitarianism\n\nPeter Singer (b. 1946) defends a preference utilitarianism that extends moral consideration to all sentient beings capable of having preferences.^[Singer, P. (1979). *Practical Ethics*. Cambridge University Press.] The boundary of the moral community is not species membership but sentience. Singer's argument for animal liberation, global poverty obligations, and euthanasia all follow from applying the impartial preference calculus rigorously.\n\nSinger's *drowning child* argument: if you could save a drowning child at trivial cost to yourself, you are morally required to do so. But the same logic applies to distant strangers dying of preventable diseases. If distance does not diminish moral obligation, affluent people in wealthy nations are obligated to give dramatically more than they typically do.^[Singer, P. (1972). \"Famine, Affluence, and Morality.\" *Philosophy & Public Affairs*, 1(3).]\n\n## Objections\n\n### The Demandingness Objection\nImpartial consequentialism seems to demand that we sacrifice almost all personal projects, relationships, and pleasures to maximise aggregate welfare. Bernard Williams argued that this alienates us from our own \"ground projects\" โ€” the commitments that give our lives meaning.^[Williams, B. (1973). \"A Critique of Utilitarianism.\" In Smart & Williams, *Utilitarianism: For and Against*.]\n\n### The Integrity Objection\nWilliams' related argument: if consequences are all that matter, then I should be willing to perform any act โ€” including acts I find deeply repugnant โ€” if doing so maximises welfare. This seems to demand that agents violate their own integrity in ways that undermine the coherence of a moral life.\n\n### The Measurement Problem\nHow do we compare welfare across persons? Cardinal welfare comparisons are notoriously difficult. Preference satisfaction is a proxy, but preferences can be adaptive (the oppressed learn to desire less), malformed, or satisfied in ways that harm the agent.\n\n### Rights Violations\nRobert Nozick's side-constraints view: there are moral side-constraints on action โ€” rights โ€” that cannot be overridden even by sufficiently large welfare gains. Using a person merely as a means to aggregate welfare violates their dignity as an end in themselves.^[Nozick, R. (1974). *Anarchy, State, and Utopia*. Basic Books.]\n\n## Sophisticated Consequentialism\n\nMany contemporary consequentialists have developed more sophisticated positions that accommodate common moral intuitions:\n\n- **Indirect consequentialism**: evaluate character traits and dispositions by their consequences, not individual acts\n- **Two-level utilitarianism** (Hare): intuitive level rules for everyday decision-making, critical level for theoretical reflection\n- **Satisficing consequentialism**: require producing good-enough outcomes rather than maximising\n\nThese refinements preserve the spirit of consequentialism while avoiding the most counterintuitive implications.\n\n## Further Reading\n\n- Parfit, D. (1984). *Reasons and Persons*. Oxford University Press.\n- Crisp, R. (1997). *Mill on Utilitarianism*. Routledge.\n- Kagan, S. (1989). *The Limits of Morality*. Oxford University Press." + }, + { + "file": "pages/eth-03-deontology.md", + "title": "Deontological Ethics", + "section-id": "ethics", + "keywords": "", + "description": "Kant's categorical imperative, the formulas of universal law and humanity, perfect and imperfect duties, and neo-Kantian developments.", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Deontological Ethics\n\nDeontological ethics holds that certain actions are intrinsically right or wrong regardless of their consequences. The term derives from the Greek *deon* (duty). Immanuel Kant (1724โ€“1804) constructed the most influential deontological system in the history of ethics, grounding morality entirely in reason rather than sentiment or consequences.\n\n## Kant's Moral Philosophy: Starting Points\n\nKant begins the *Groundwork of the Metaphysics of Morals* (1785) by identifying the only thing that is good without qualification: a **good will**.^[Kant, I. (1785). *Groundwork of the Metaphysics of Morals*. Trans. Korsgaard, Cambridge UP, 1998.] Intelligence, courage, and even happiness can be used for evil purposes. But a will that acts from duty โ€” that acts because doing so is right, regardless of inclination or consequence โ€” is good unconditionally.\n\nThis distinguishes acting *in accordance with* duty (which a prudent merchant might do for self-interested reasons) from acting *from* duty (the only source of genuine moral worth).\n\n## The Categorical Imperative\n\nKant argues that all genuine moral requirements are *categorical* imperatives โ€” commands that apply unconditionally, regardless of one's desires. \"Pay your debts\" is categorical: it applies whether or not you want to, whether or not it benefits you. By contrast, \"If you want to be trusted, pay your debts\" is a *hypothetical* imperative, binding only if you have the relevant desire.\n\nKant offers three principal formulations of the categorical imperative, claiming they are equivalent:\n\n### Formula of Universal Law (FUL)\n> \"Act only according to that maxim whereby you can at the same time will that it should become a universal law.\"\n\nTo test whether an action is permissible, extract the maxim (underlying principle) of the action and ask: could I consistently will that everyone act on this maxim? The classic example is lying promises. My maxim: \"When in financial difficulty, I will make a false promise to repay a loan.\" If universalised, the institution of promising collapses โ€” no one would believe promises. The maxim is self-defeating when universalised.\n\n### Formula of Humanity (FH)\n> \"Act so that you treat humanity, whether in your own person or in that of another, always as an end and never as a means only.\"\n\nPersons have *dignity* โ€” a value beyond all price. Using someone merely as an instrument for your purposes violates their status as a rational, self-legislating agent. This formula generates more intuitive verdicts than FUL in many cases and grounds a robust conception of human rights.\n\n### Formula of the Kingdom of Ends (FKE)\n> \"Act according to maxims of a universally legislating member of a merely possible kingdom of ends.\"\n\nThe moral community is a hypothetical kingdom of rational agents who legislate universal laws for themselves and for all. Each person is both subject to and author of the moral law.\n\n## Perfect and Imperfect Duties\n\nKant distinguishes **perfect duties** (negative, admitting no exceptions) from **imperfect duties** (positive, allowing latitude in how they are fulfilled).\n\n- *Perfect duties*: Do not murder. Do not lie. Do not make false promises. These admit no exceptions.\n- *Imperfect duties*: Develop your talents. Help others in need. We must pursue these ends, but have discretion in how.\n\n## The Formula of Universal Law: Applications\n\nKant tests four cases:\n\n1. **Suicide to escape suffering** โ€” The maxim of self-destruction from self-love contradicts itself when universalised (life-preserving instinct cannot simultaneously mandate destroying life).\n2. **False promises** โ€” Universalised, this destroys the institution of promising.\n3. **Neglecting one's talents** โ€” Although we can consistently will a world where all neglect their talents, we cannot *rationally* will such a world as members who might need others' developed capacities.\n4. **Refusing to aid others** โ€” We cannot rationally will a world with no mutual aid, since we might need it ourselves.\n\n## Objections to Kantian Ethics\n\n### The Problem of Conflicting Duties\nWhat if telling the truth would lead to murder? The notorious example: a murderer asks you where your friend is hiding. Kant's strict application of FUL seems to require telling the truth.^[Kant, I. (1797). \"On a Supposed Right to Lie from Philanthropy.\"] Most critics find this conclusion intolerable. Defenders argue Kant was wrong to apply his own theory in this case.\n\n### The Formalism Objection (Hegel)\nHegel objected that the categorical imperative is empty โ€” too formal to generate determinate moral content. Almost any maxim can be made consistent with FUL through reformulation.^[Hegel, G.W.F. (1821). *Philosophy of Right*, ยง135.]\n\n### The Rigorism Objection\nThe absolute prohibition on lying, even to prevent serious harm, seems morally obtuse. A moral theory that ignores consequences entirely cannot be adequate.\n\n### The Humanity Formula and Its Scope\nDoes FH extend to animals? Kant seems to deny that animals have dignity (since they lack rationality), but this generates counterintuitive implications about the permissibility of animal cruelty.\n\n## Neo-Kantian Developments\n\n**Christine Korsgaard** grounds Kantian ethics in the structure of reflective self-consciousness. When we act, we implicitly endorse a principle. Practical identity โ€” the source of all our obligations โ€” commits us to valuing humanity as an end.^[Korsgaard, C. (1996). *Sources of Normativity*. Cambridge University Press.]\n\n**Thomas Scanlon's contractualism**: An act is wrong if its performance under the circumstances would be disallowed by any set of principles that no one could reasonably reject.^[Scanlon, T.M. (1998). *What We Owe to Each Other*. Harvard University Press.] This grounds moral requirements in what we owe to each other as persons โ€” a broadly Kantian spirit without the metaphysical apparatus.\n\n**W.D. Ross** introduced the concept of *prima facie* duties โ€” duties that are binding unless overridden by stronger competing duties in a given situation. Fidelity, gratitude, non-maleficence, beneficence, and justice are among them. This pluralist deontology avoids the single-minded rigour of Kant while preserving the idea that some actions have moral weight independent of consequences.^[Ross, W.D. (1930). *The Right and the Good*. Oxford University Press.]\n\n## Further Reading\n\n- Korsgaard, C. (1996). *Creating the Kingdom of Ends*. Cambridge University Press.\n- O'Neill, O. (1989). *Constructions of Reason*. Cambridge University Press.\n- Herman, B. (1993). *The Practice of Moral Judgment*. Harvard University Press." + }, + { + "file": "pages/eth-04-virtue.md", + "title": "Virtue Ethics", + "section-id": "ethics", + "keywords": "", + "description": "Aristotle's eudaimonia, the virtues, the doctrine of the mean, and contemporary neo-Aristotelian revival in moral philosophy.", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Virtue Ethics\n\nVirtue ethics shifts the primary question of moral theory from \"What should I do?\" to \"What kind of person should I be?\" Rather than specifying rules or calculating consequences, virtue ethics focuses on the character traits โ€” the *virtues* โ€” that constitute human excellence and the good life.\n\n## Aristotle's Ethics\n\nThe foundational text is Aristotle's *Nicomachean Ethics* (ca. 350 BCE).^[Aristotle. *Nicomachean Ethics*. Trans. Irwin, Hackett, 1999.] Aristotle begins with the observation that every action aims at some good. The highest good โ€” the good for its own sake โ€” he calls *eudaimonia*, usually translated as \"happiness\" but better rendered as *flourishing* or *living well*.\n\nEudaimonia is not a feeling but an *activity*: the activity of the soul in accordance with virtue (*arete*). It is not a momentary state but characterises a complete life.\n\n## The Function Argument\n\nAristotle argues that just as a knife has a function (cutting) and a good knife fulfils its function excellently, human beings have a characteristic function. The human function, unique among animals, is rational activity. *Eudaimonia* is the excellent exercise of our rational capacities.^[*NE* I.7, 1097b24โ€“1098a20.]\n\nThis *ergon* (function) argument has been criticised for assuming that humans have a single, discoverable function. Nonetheless, it provides the teleological framework within which the virtues are defined.\n\n## The Doctrine of the Mean\n\nVirtues are stable dispositions of character that enable us to respond appropriately to situations. They are acquired through habituation: we become courageous by practising courageous acts. The virtuous person does not merely act rightly but does so with pleasure and without painful struggle โ€” virtue has been fully internalised.\n\nEach virtue is a **mean** (*mesotes*) between two extremes โ€” excess and deficiency. Courage is the mean between cowardice (deficiency of boldness) and rashness (excess). Generosity lies between miserliness and prodigality. The mean is not arithmetically fixed but *relative to us* โ€” what counts as appropriate depends on context and the individual.\n\n| Excess | Virtue | Deficiency |\n|---|---|---|\n| Rashness | Courage | Cowardice |\n| Prodigality | Generosity | Miserliness |\n| Vanity | Magnanimity | Pusillanimity |\n| Obsequiousness | Friendliness | Quarrelsomeness |\n| Buffoonery | Wit | Boorishness |\n\n## Practical Wisdom: Phronesis\n\nThe master virtue in Aristotle's scheme is *phronesis* โ€” practical wisdom. The person of practical wisdom perceives what a situation requires, deliberates well about how to act, and acts accordingly. Practical wisdom is not reducible to following rules; it requires experience, perception, and judgment.\n\nThis distinguishes virtue ethics from rule-based approaches: no finite set of rules can capture what the practically wise person knows. The virtuous person's perceptions and responses are *constitutive* of right action, not mere applications of antecedent principles.\n\n## The Unity of the Virtues\n\nAristotle holds that the virtues are unified: one cannot genuinely have any virtue without practical wisdom, and practical wisdom requires all the virtues. This *unity thesis* is controversial โ€” it seems possible to be courageous but unjust. Defenders argue that only with the full integration of virtues does one have the \"complete\" versions; partial virtues are mere natural tendencies, not fully-fledged character traits.\n\n## The Neo-Aristotelian Revival\n\nVirtue ethics experienced a significant revival in twentieth-century analytic philosophy, partly as a reaction to the perceived limitations of both consequentialism and deontology.\n\n**G.E.M. Anscombe** (1958) argued that concepts like \"moral obligation\" and \"duty\" are residues of a divine-law framework that has been abandoned; without God, they are incoherent. We should return to Aristotelian concepts of virtue, human nature, and flourishing.^[Anscombe, G.E.M. (1958). \"Modern Moral Philosophy.\" *Philosophy*, 33(124).]\n\n**Philippa Foot** developed a naturalistic virtue ethics grounding virtues in what is good for humans as the kind of organisms we are. *Natural goodness* is a matter of the proper functioning of organisms of a particular natural kind.^[Foot, P. (2001). *Natural Goodness*. Oxford University Press.]\n\n**Alasdair MacIntyre** (*After Virtue*, 1981) argued that contemporary moral discourse is fragmented and incoherent because we have lost the teleological framework within which virtue concepts made sense. The Enlightenment project of grounding morality in individual reason or sentiment was doomed to fail. We need to recover Aristotelian tradition โ€” structured around practices, narrative, and community โ€” to make ethics intelligible.^[MacIntyre, A. (1981). *After Virtue*. University of Notre Dame Press.]\n\n## Contemporary Virtue Ethics\n\n**Rosalind Hursthouse** has developed a virtue-theoretic account of right action: an action is right if it is what a virtuous person would characteristically do in the circumstances.^[Hursthouse, R. (1999). *On Virtue Ethics*. Oxford University Press.] This need not be circular: virtuous persons are those with character traits that constitute human flourishing, and we can characterise flourishing independently.\n\n**Michael Slote** defends agent-based virtue ethics, grounding moral evaluation entirely in the motivational states of agents rather than objective human flourishing.\n\n**Julia Annas** argues that virtue ethics is best understood not as a rival to rule-following but as an account of how we internalise moral requirements through the development of character.\n\n## Objections to Virtue Ethics\n\n### Action Guidance\nCritics allege that virtue ethics provides insufficient guidance: when I face a difficult choice, \"act as a virtuous person would\" tells me little unless I already know what virtue requires. The response is that moral life is not primarily about making hard decisions but about forming character โ€” and character provides guidance of a richer kind than any algorithm.\n\n### Cultural Relativism\nIf virtues are defined relative to a human *telos* and that *telos* varies across cultures, different cultures will have different, potentially incompatible, lists of virtues. MacIntyre acknowledges this but argues that tradition-internal reasoning can achieve cross-traditional rational dialogue.\n\n### The Problem of the Selfish Gene\nIf we are products of natural selection, and selection favours genes that promote reproductive fitness, then \"human nature\" is not a stable, rationally accessible guide to flourishing. The naturalistic programme of Foot and Hursthouse faces this challenge.\n\n## Further Reading\n\n- Annas, J. (2011). *Intelligent Virtue*. Oxford University Press.\n- Crisp, R. and Slote, M. (eds.) (1997). *Virtue Ethics*. Oxford University Press.\n- Williams, B. (1985). *Ethics and the Limits of Philosophy*. Fontana." + }, + { + "file": "pages/eth-05-applied.md", + "title": "Applied Ethics", + "section-id": "ethics", + "keywords": "", + "description": "Ethical theory in practice โ€” bioethics, AI ethics, environmental ethics, and the methodology of applying philosophical principles to real-world problems.", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Applied Ethics\n\nApplied ethics brings philosophical theory to bear on concrete moral problems โ€” questions arising in medicine, technology, environmental policy, business, and law. The movement gained momentum in the 1960s and 1970s, driven partly by rapid advances in medicine and biotechnology that created novel moral dilemmas for which traditional frameworks offered insufficient guidance.\n\n## The Methodology of Applied Ethics\n\nApplied ethics is not mere application of theory to cases, as if ethical systems were algorithms waiting to be run. Several methodological approaches exist:\n\n**Top-down application**: Begin with an ethical theory (utilitarian, Kantian), derive principles, apply to cases. The limitation is that theories are contested; disagreement about foundations propagates into applied questions.\n\n**Case-based reasoning (casuistry)**: Begin with clear, paradigm cases where moral judgement is confident, then reason analogically to harder cases. Associated with clinical ethics consultation. The limitation is that paradigm cases must eventually be justified by something more than intuition.\n\n**Reflective equilibrium**: Move iteratively between principles and case judgements, revising both until achieving coherence. The approach most widely used in practice.\n\n**Specification**: Take general principles (do not harm, respect autonomy) and progressively specify them to handle particular cases without derivation from a complete theory.\n\n## Bioethics\n\nBioethics addresses moral questions arising in medicine and biological research. The *Georgetown mantra* โ€” four principles identified by Beauchamp and Childress โ€” has become the dominant framework in clinical practice:^[Beauchamp, T. and Childress, J. (2019). *Principles of Biomedical Ethics*, 8th ed. Oxford University Press.]\n\n1. **Autonomy** โ€” Respect for the patient's self-determination. Informed consent is the operational expression of this principle. Patients with decision-making capacity may refuse treatment, even life-saving treatment.\n2. **Beneficence** โ€” Act in the patient's best interests. Not merely \"do no harm\" but actively promote welfare.\n3. **Non-maleficence** โ€” *Primum non nocere* (first, do no harm). Avoid imposing risks disproportionate to benefits.\n4. **Justice** โ€” Fair distribution of benefits, risks, and burdens. Includes both procedural fairness and distributive justice in healthcare resource allocation.\n\n### End-of-Life Ethics\n\nThe moral permissibility of assisted dying โ€” physician-assisted suicide (PAS) and voluntary euthanasia โ€” is among the most contested issues in bioethics. Arguments in favour appeal to autonomy: competent patients should determine the manner and timing of their deaths. Arguments against cite concerns about palliative care, the potential for coercion, and the symbolic significance of medical killing for the doctor-patient relationship.\n\nPeter Singer and James Rachels defend active euthanasia, arguing that the distinction between killing and letting die is morally irrelevant when intentions and outcomes are the same.^[Rachels, J. (1975). \"Active and Passive Euthanasia.\" *New England Journal of Medicine*, 292(2).] Opponents invoke the doctrine of double effect and the integrity of the medical profession.\n\n### Research Ethics\n\nThe Nuremberg Code (1947) and the Declaration of Helsinki (1964) emerged from scandals of medical experimentation on non-consenting subjects. Core requirements: voluntary informed consent, scientific validity, favourable risk-benefit ratio, and independent ethical review. The Belmont Report (1979) added justice as a requirement โ€” the burdens and benefits of research must be distributed fairly.\n\n## AI and Technology Ethics\n\nThe rapid development of artificial intelligence creates a new domain for applied ethics. Key issues include:\n\n**Algorithmic bias**: Machine learning systems trained on historical data can encode and amplify existing discriminatory patterns. A loan approval algorithm trained on historical lending data may perpetuate racial discrimination without any discriminatory intent. Fairness criteria (demographic parity, equalised odds, calibration) are mathematically incompatible in general โ€” we cannot satisfy all simultaneously.^[Chouldechova, A. (2017). \"Fair Prediction with Disparate Impact.\" *Big Data*, 5(2).]\n\n**Autonomous systems**: When an autonomous vehicle must choose between killing one pedestrian or five, how should it be programmed? Trolley-problem style dilemmas in algorithmic form raise questions about whether utilitarian calculus should be codified into machines, and who bears moral responsibility for automated decisions.\n\n**AI consciousness and moral status**: If future AI systems develop something like sentience or interests, do they merit moral consideration? The philosophical difficulty of consciousness (see the chapter on philosophy of mind) makes this question genuinely hard.\n\n**Privacy and surveillance**: The collection of vast personal data by states and corporations raises questions about informational privacy as an aspect of autonomy and dignity. Nissenbaum's concept of *contextual integrity* โ€” information flows respect privacy when they match the norms of the context in which they were generated โ€” provides a useful framework.\n\n## Environmental Ethics\n\nTraditional ethics is anthropocentric โ€” it recognises moral obligations only to persons. Environmental ethics asks: do non-human animals, species, ecosystems, or nature as a whole have intrinsic moral value?\n\n**Animal ethics**: Peter Singer's utilitarian case for animal liberation grounds obligations in the capacity for suffering: if pain is bad for humans, it is bad for pigs, who suffer equally.^[Singer, P. (1975). *Animal Liberation*. New York Review Books.] Tom Regan's rights-based account argues that animals who are \"subjects of a life\" โ€” with beliefs, desires, and a welfare โ€” have inherent value that may not be traded off against aggregate utility.^[Regan, T. (1983). *The Case for Animal Rights*. University of California Press.]\n\n**Biocentric ethics** (Paul Taylor): Every living organism has a good of its own that commands moral respect. We have prima facie duties not to harm living things, override-able only for weighty reasons.^[Taylor, P. (1986). *Respect for Nature*. Princeton University Press.]\n\n**Ecocentric ethics**: Aldo Leopold's *land ethic* โ€” \"A thing is right when it tends to preserve the integrity, stability, and beauty of the biotic community\" โ€” extends moral consideration to ecosystems and species, not just individuals.^[Leopold, A. (1949). *A Sand County Almanac*. Oxford University Press.]\n\n**Climate ethics** raises questions about intergenerational justice (obligations to future persons), international justice (who bears the costs of mitigation), and the ethics of geoengineering.\n\n## Further Reading\n\n- Rachels, J. and Rachels, S. (2019). *The Elements of Moral Philosophy*, 9th ed. McGraw-Hill.\n- Jamieson, D. (2014). *Reason in a Dark Time: Why the Struggle Against Climate Change Failed*. Oxford University Press.\n- Floridi, L. (ed.) (2015). *The Onlife Manifesto*. Springer." + }, + { + "file": "pages/eth-06-political.md", + "title": "Political Philosophy", + "section-id": "ethics", + "keywords": "", + "description": "Rawls, Nozick, communitarianism, and contemporary debates about justice, liberty, and the legitimate authority of the state.", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Political Philosophy\n\nPolitical philosophy investigates the normative foundations of political institutions: the state, law, political authority, rights, and justice. Its central questions include: What justifies political authority? What makes a distribution of benefits and burdens just? What are the limits of individual liberty? What do citizens owe one another?\n\n## The Social Contract Tradition\n\nThe dominant tradition in modern political philosophy grounds political authority in a *social contract* โ€” an actual or hypothetical agreement among individuals to establish political institutions. The tradition includes Hobbes, Locke, and Rousseau, but it is John Rawls who gave it its most sophisticated contemporary form.\n\n### Hobbes\nThomas Hobbes (1651) argued that without political authority, life would be \"solitary, poor, nasty, brutish, and short.\"^[Hobbes, T. (1651). *Leviathan*, Ch. XIII.] Rational agents in the state of nature would contract into an absolute sovereign to secure peace. Hobbes's argument is primarily consequentialist in structure: sovereignty is justified by the order it creates.\n\n### Locke\nJohn Locke (1689) grounded political authority in natural rights โ€” rights to life, liberty, and property that individuals possess prior to and independently of the state. Government is legitimate only if it protects these rights; when it systematically violates them, citizens have a right of revolution. Locke's theory provides the philosophical basis for liberal constitutionalism.^[Locke, J. (1689). *Two Treatises of Government*, Second Treatise.]\n\n## Rawls's Theory of Justice\n\nJohn Rawls (*A Theory of Justice*, 1971) represents the most influential work in twentieth-century political philosophy. Rawls aims to identify principles of justice that free and rational persons would accept in an original position of equality.\n\n### The Original Position and the Veil of Ignorance\n\nThe *original position* is a hypothetical decision procedure. We imagine choosing principles of justice behind a *veil of ignorance*: we do not know our place in society, our class position, our natural abilities, our conception of the good, or the generation we belong to. This ensures impartiality โ€” no one can tailor principles to benefit their particular position.^[Rawls, J. (1971). *A Theory of Justice*. Harvard University Press, ยง3.]\n\nFrom behind the veil, Rawls argues, rational agents would choose two principles:\n\n1. **The Equal Liberty Principle**: Each person is to have an equal right to the most extensive system of equal basic liberties compatible with a similar system of liberty for all.\n2. **The Difference Principle**: Social and economic inequalities are to be arranged so that they are: (a) attached to offices and positions open to all under fair equality of opportunity, and (b) to the greatest benefit of the least advantaged members of society.\n\nThe principles are *lexically ordered*: the first has absolute priority over the second. Basic liberties cannot be traded off against economic gains.\n\n### The Difference Principle\n\nThe Difference Principle is Rawls's most distinctive and controversial contribution. It permits inequalities only if they maximally benefit the worst-off group. This *maximin* strategy โ€” maximise the minimum position โ€” is what rational agents under uncertainty would choose, according to Rawls.\n\nThe argument: since I do not know whether I will be advantaged or disadvantaged, and since the stakes (basic life prospects) are very high, rationality demands choosing the arrangement that makes the worst-case scenario as good as possible.\n\n## Nozick's Libertarianism\n\nRobert Nozick (*Anarchy, State, and Utopia*, 1974) offers a forceful alternative.^[Nozick, R. (1974). *Anarchy, State, and Utopia*. Basic Books.] Beginning from strong Lockean natural rights โ€” that individuals may not be used against their will as means to others' ends โ€” Nozick argues that only a minimal state (limited to protecting against force and fraud) is justified.\n\nAny more extensive state violates individual rights. Redistributive taxation, for Nozick, is morally equivalent to forced labour: it takes the product of a person's labour and transfers it to others without consent.\n\nNozick's *entitlement theory* of justice: a distribution is just if it arises from just original acquisitions and just transfers. Historical process, not distributional pattern, determines justice. No patterned principle โ€” whether egalitarian or utilitarian โ€” can be maintained without continuous interference with free exchange.\n\n### Wilt Chamberlain Argument\nNozick's celebrated argument: suppose we start from any distribution D1 you consider just. Wilt Chamberlain, a basketball star, charges fans 25 cents per game to see him play. One million fans freely pay. Now Chamberlain has $250,000 more than D1 allowed. Is D2 unjust? But it arose through voluntary exchanges from a just starting point. Any patterned theory must continuously prohibit voluntary exchanges โ€” and this is incompatible with liberty.\n\n## Communitarianism\n\nIn the 1980s, a group of philosophers โ€” Michael Sandel, Charles Taylor, Alasdair MacIntyre, and Michael Walzer โ€” challenged liberalism's conception of the self and the priority it gives to justice over the good.\n\n**Sandel's critique of the unencumbered self**: Rawls's original position presupposes a self that is prior to and independent of its ends and social roles. But this is incoherent: we cannot abstract ourselves from the constitutive attachments and community memberships that make us who we are. A more adequate political philosophy would recognise that we are *embedded* in communities whose values and traditions define our identities.^[Sandel, M. (1982). *Liberalism and the Limits of Justice*. Cambridge University Press.]\n\n**Walzer's complex equality**: Justice requires that different social goods โ€” money, political power, medical care, education โ€” be distributed according to their own internal norms, not reduced to a single metric. Injustice is not inequality per se but *dominance*: when one good (typically money) is used to control access to all others.^[Walzer, M. (1983). *Spheres of Justice*. Basic Books.]\n\n## Global Justice\n\nRawls's *The Law of Peoples* (1999) applied his framework internationally, but controversially limited global distributive obligations to \"duty of assistance\" to \"burdened societies.\" Cosmopolitan theorists (Thomas Pogge, Charles Beitz) argue that global economic institutions impose injustice on the world's poor, generating stringent obligations to reform them.^[Pogge, T. (2002). *World Poverty and Human Rights*. Polity Press.]\n\nThe debate between Rawlsian nationalism and cosmopolitanism turns on whether Rawlsian principles apply only within cooperative schemes (nation-states) or to all human beings as such.\n\n## Further Reading\n\n- Freeman, S. (2007). *Justice and the Social Contract*. Oxford University Press.\n- Cohen, G.A. (2008). *Rescuing Justice and Equality*. Harvard University Press.\n- Kymlicka, W. (2002). *Contemporary Political Philosophy*, 2nd ed. Oxford University Press." + }, + { + "file": "pages/further-reading.md", + "title": "Further Reading", + "section-id": "conclusion", + "keywords": "", + "description": "Annotated bibliography organised by chapter, with commentary on essential secondary texts and resources for continued study.", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Further Reading\n\nThis annotated bibliography is organised by chapter. For each topic, the most accessible introductory texts are listed first, followed by more advanced or specialised works. All items marked **[Core]** are considered essential reading; unmarked items represent productive next steps for those wishing to go deeper.\n\n---\n\n## Part I: Epistemology\n\n### What is Knowledge?\n**[Core]** Gettier, E.L. (1963). \"Is Justified True Belief Knowledge?\" *Analysis*, 23(6), 121โ€“123. โ€” Three pages that changed epistemology. Required reading.\n\n**[Core]** Chisholm, R. (1977). *Theory of Knowledge*, 2nd ed. Prentice Hall. โ€” The classic textbook on epistemological foundations.\n\nZagzebski, L. (1994). \"The Inescapability of Gettier Problems.\" *Philosophical Quarterly*, 44(174), 65โ€“73. โ€” Shows the structural depth of the problem.\n\nWilliamson, T. (2000). *Knowledge and Its Limits*. Oxford University Press. โ€” Defends knowledge as a prime epistemic concept; difficult but rewarding.\n\n### Perception and Reality\n**[Core]** Ayer, A.J. (1956). *The Problem of Knowledge*. Penguin. โ€” Accessible and wide-ranging.\n\nDancy, J. (1985). *Introduction to Contemporary Epistemology*. Blackwell. โ€” Chapter 6 on perception is particularly good.\n\nMcDowell, J. (1994). *Mind and World*. Harvard University Press. โ€” Demanding but essential for understanding the conceptualism debate.\n\n### Scepticism\n**[Core]** Descartes, R. (1641). *Meditations on First Philosophy*. โ€” The primary source; any good translation suffices.\n\nStroud, B. (1984). *The Significance of Philosophical Scepticism*. Oxford University Press. โ€” Why scepticism cannot be easily dismissed.\n\nDeRose, K. (2009). *The Case for Contextualism*. Oxford University Press.\n\n### Truth\n**[Core]** Horwich, P. (1998). *Truth*, 2nd ed. Oxford University Press. โ€” Deflationary theory, clearly argued.\n\nLynch, M. (2009). *Truth as One and Many*. Oxford University Press. โ€” Pluralist theory.\n\n---\n\n## Part II: Metaphysics\n\n### Existence and Ontology\n**[Core]** Quine, W.V.O. (1948). \"On What There Is.\" *Review of Metaphysics*, 2(5). โ€” The classic statement of Quinean ontology.\n\n**[Core]** van Inwagen, P. (1998). \"Meta-Ontology.\" *Erkenntnis*, 48(2โ€“3), 233โ€“250.\n\nThomasson, A. (2015). *Ontology Made Easy*. Oxford University Press. โ€” Deflationary approach; valuable counterpoint to heavyweight ontology.\n\n### Identity and Persistence\n**[Core]** Parfit, D. (1984). *Reasons and Persons*, Part III. Oxford University Press. โ€” The most influential modern treatment.\n\nLewis, D. (1976). \"Survival and Identity.\" In Rorty, A. (ed.), *The Identities of Persons*. Berkeley.\n\nOlson, E. (1997). *The Human Animal*. Oxford University Press. โ€” Animalist view.\n\n### Free Will\n**[Core]** Kane, R. (1996). *The Significance of Free Will*. Oxford University Press. โ€” Best defence of libertarianism.\n\n**[Core]** Frankfurt, H. (1969). \"Alternate Possibilities and Moral Responsibility.\" *Journal of Philosophy*, 66(23). โ€” Frankfurt cases; only five pages, transformative.\n\nStrawson, P.F. (1962). \"Freedom and Resentment.\" *Proceedings of the British Academy*, 48. โ€” Foundational compatibilist paper.\n\nFischer, J.M. and Ravizza, M. (1998). *Responsibility and Control*. Cambridge University Press.\n\n### Philosophy of Mind\n**[Core]** Nagel, T. (1974). \"What Is It Like to Be a Bat?\" *Philosophical Review*, 83(4). โ€” The classic statement of the explanatory gap.\n\n**[Core]** Chalmers, D. (1996). *The Conscious Mind*. Oxford University Press. โ€” Comprehensive case for the hard problem.\n\nDennett, D. (1991). *Consciousness Explained*. Little, Brown. โ€” The physicalist response; readable and provocative.\n\nJackson, F. (1986). \"What Mary Didn't Know.\" *Journal of Philosophy*, 83(5). โ€” Knowledge argument in five pages.\n\n---\n\n## Part III: Ethics\n\n### Metaethics\n**[Core]** Mackie, J.L. (1977). *Ethics: Inventing Right and Wrong*. Penguin. โ€” Error theory; accessible and well-argued.\n\nBlackburn, S. (1998). *Ruling Passions*. Oxford University Press. โ€” Best recent defence of quasi-realism.\n\nEnoch, D. (2011). *Taking Morality Seriously*. Oxford University Press. โ€” Strong defence of robust moral realism.\n\n### Consequentialism\n**[Core]** Mill, J.S. (1863). *Utilitarianism*. โ€” Short; read in an afternoon; annotated editions recommended.\n\n**[Core]** Singer, P. (1979). *Practical Ethics*. Cambridge University Press. โ€” Applies utilitarian reasoning to live issues.\n\nParfit, D. (1984). *Reasons and Persons*, Part IV. โ€” \"Repugnant Conclusion\" and population ethics; essential.\n\n### Deontological Ethics\n**[Core]** Kant, I. (1785). *Groundwork of the Metaphysics of Morals*. Trans. Korsgaard. Cambridge UP. โ€” Use Korsgaard's translation and commentary.\n\n**[Core]** Ross, W.D. (1930). *The Right and the Good*, Chs. 1โ€“2. Oxford University Press.\n\nScanlon, T.M. (1998). *What We Owe to Each Other*. Harvard University Press. โ€” Contractualist deontology; rich and rewarding.\n\n### Virtue Ethics\n**[Core]** Aristotle. *Nicomachean Ethics*, Books Iโ€“II, X. โ€” Any good translation.\n\n**[Core]** Hursthouse, R. (1999). *On Virtue Ethics*. Oxford University Press.\n\nMacIntyre, A. (1981). *After Virtue*. University of Notre Dame Press. โ€” Polemical and influential.\n\n### Political Philosophy\n**[Core]** Rawls, J. (1971). *A Theory of Justice*. Harvard University Press. โ€” Read at minimum Part I.\n\n**[Core]** Nozick, R. (1974). *Anarchy, State, and Utopia*. Basic Books. โ€” Especially Ch. 7 on distributive justice.\n\nKymlicka, W. (2002). *Contemporary Political Philosophy*, 2nd ed. Oxford University Press. โ€” Best survey text.\n\n---\n\n## General Philosophy Reference\n\n**Stanford Encyclopedia of Philosophy** (plato.stanford.edu) โ€” Freely available online; peer-reviewed, regularly updated. An invaluable first resource for any philosophical topic.\n\n**[Core]** Blackburn, S. (1996). *Oxford Dictionary of Philosophy*, 3rd ed. Oxford University Press. โ€” Concise, reliable reference for key terms.\n\nCraig, E. (ed.) (1998). *Routledge Encyclopedia of Philosophy*. โ€” 10-volume scholarly reference.\n\n---\n\n## On Reading Philosophy\n\nPhilosophical texts reward rereading. A text that seems clear at first glance often conceals assumptions that become visible only on the third or fourth reading. Keep a running list of assumptions, note where each argument depends on undefended premises, and always ask: what would need to be true for this argument to fail?\n\nDiscussion and disagreement are essential. Read with a philosophical friend or in a seminar. The objections you make and receive in conversation will teach you more than any further reading." + }, + { + "file": "pages/how-to-use.md", + "title": "How to Use This Book", + "section-id": "front-matter", + "keywords": "", + "description": "Reading guide, chapter dependencies, glossary note, and further reading approach.", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "![A well-stocked research library](assets/images/library.jpg)\n\n# How to Use This Book\n\nThis book is designed to be accessible to readers coming from different starting points and with different purposes. What follows is a brief guide to getting the most from it.\n\n## Sequence and Structure\n\nThe book is organised into three parts โ€” Epistemology, Metaphysics, and Ethics โ€” each comprising six chapters. The parts are designed to be read in order, since later discussions often presuppose earlier material. The epistemology chapters, for instance, establish vocabulary and conceptual distinctions (justification, a priori knowledge, reliabilism, coherentism) that recur throughout the metaphysics and ethics discussions.\n\nWithin each part, the chapters proceed from foundational questions toward more specific and applied ones. In epistemology, we begin with the basic analysis of knowledge before examining specific faculties (perception, reason) and specific challenges (scepticism). In ethics, we examine the foundations of moral inquiry before turning to the major normative theories and then applied questions.\n\nThat said, the book is cross-referenced and many chapters can be read independently by a reader who already has some background. The chapter on free will and determinism (Part II, Chapter 4) can be read alongside the epistemology chapter on scepticism and the ethics chapter on moral responsibility, and is usefully paired with the applied ethics chapter's discussion of criminal justice. The chapter on philosophy of mind (Part II, Chapter 5) is closely connected to the epistemology discussion of perception and knowledge.\n\n## Chapter Structure\n\nEach chapter follows a common pattern:\n\n1. **Introduction** โ€” orienting the problem, explaining why it matters\n2. **Main positions** โ€” surveyed with their central arguments\n3. **Key arguments and objections** โ€” examined with care\n4. **Connections and implications** โ€” how the debate bears on other questions\n5. **Inline footnotes** โ€” key citations in the form `^[Author, Year, p.X]`\n\nI have used inline footnotes throughout rather than endnotes. Philosophy students need to learn to read with citations โ€” to understand that positions have sources, that arguments have authors, and that intellectual honesty requires acknowledging where ideas come from.\n\n## The Primary Sources\n\nThis is an introductory text. It cannot substitute for reading primary sources, and it is not intended to. The goal is to prepare you to read Descartes, Hume, Kant, and their successors with understanding โ€” to give you the context and vocabulary to follow their arguments, so that when you turn to the *Meditations* or the *Enquiry* or the *Groundwork*, you know what question you are supposed to be thinking about.\n\nAt the end of the book you will find an annotated bibliography organised by chapter. Each entry includes a note on what the text contributes and who it is most suitable for. Use it as a starting point for primary reading, not as a substitute for it.\n\n## A Note on Difficulty\n\nPhilosophy is hard. It requires holding complex arguments in memory while evaluating their steps, tracking distinctions that initially seem subtle and become crucial, and maintaining intellectual patience through periods of genuine uncertainty. This is not a reason to find the difficulty discouraging โ€” it is a reason to take it seriously.\n\nWhen you find yourself confused, the first question to ask is not \"Am I misunderstanding the argument?\" but \"Is the argument actually this hard?\" Sometimes the answer to the second question is yes, and the apparent confusion reflects something genuinely difficult in the material. At other times, a re-reading or a change of perspective resolves things.\n\nIt is worth keeping notes as you read โ€” writing down the main claim of each section, the central argument, and your own questions and objections. Philosophy is better done with a pen in hand than as a purely passive activity.\n\n## On Philosophical Writing\n\nStudents who will be writing essays in philosophy should note that philosophical writing is argument, not assertion. The task is not to state what you believe but to give reasons for believing it, to consider and respond to objections, and to be honest about what you do not yet know.\n\nGood philosophical writing is clear, precise, and fair to opposing views. It uses technical vocabulary accurately โ€” not to impress, but because precision matters and ordinary language is often insufficiently precise for philosophical work. It acknowledges the difficulty of the questions rather than pretending to more certainty than is warranted.\n\nThese are not stylistic preferences. They are requirements of intellectual honesty, and they are what distinguish good philosophical writing from merely asserting things confidently.\n\n## Glossary\n\nTechnical terms are defined when first introduced and collected in the index with page references. Key terms include: *a priori*, *a posteriori*, *analytic*, *synthetic*, *justified true belief*, *internalism*, *externalism*, *substance*, *property*, *supervenience*, *compatibilism*, *libertarianism* (in the metaphysical sense), *hard determinism*, *consequentialism*, *deontology*, *virtue ethics*, *metaethics*, *normative ethics*.\n\nDo not be discouraged if these terms initially seem arbitrary. They acquire meaning through the arguments that use them, and they become tools rather than obstacles once you have enough context to understand why the distinctions they mark are important.\n\nGood reading." + }, + { + "file": "pages/meta-01-existence.md", + "title": "Existence and Being", + "section-id": "metaphysics", + "keywords": "", + "description": "Ontology basics, Quine's criterion of ontological commitment, existence as a predicate, and Meinong's jungle.", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Existence and Being\n\nMetaphysics is the study of the most fundamental features of reality โ€” what exists, what kinds of things exist, and what it is for something to exist at all. Ontology, its central division, addresses the question: *what is there?*\n\nThe question sounds trivially answerable: there is everything there is, and nothing else. Quine gave this answer in a sentence: \"To be is to be the value of a variable\" ^[Quine, W.V.O., \"On What There Is\", *Review of Metaphysics* 2, 1948]. But behind this slogan lies a substantial and contested philosophical methodology.\n\n## Quine's Criterion of Ontological Commitment\n\nQuine argued that the ontological commitments of a theory are revealed by *regimentation* โ€” translating the theory into first-order predicate logic and examining what the variables in the existential quantifiers must range over ^[Quine, W.V.O., \"On What There Is\"; see also *Word and Object*, MIT Press, 1960, ch.7].\n\nIf a true theory says \"There are prime numbers between 10 and 20,\" and the best regimentation of this claim quantifies over numbers, then the theory is committed to the existence of numbers. One cannot simultaneously assert a theory and deny the existence of entities the theory quantifies over.\n\nThe Quinean approach transformed ontology from a speculative metaphysical exercise into a semi-technical discipline: the question \"Does X exist?\" becomes \"Does our best overall theory of the world require quantifying over Xs?\" This is ontology anchored to science.\n\n**The criterion of ontological commitment** is: a theory is ontologically committed to those entities that, when the theory is put in canonical notation, the bound variables must range over if the theory is true.\n\n## Existence as a Predicate\n\nCan existence be predicated of individuals? Kant famously argued that existence is not a real predicate โ€” it adds nothing to the concept of a thing ^[Kant, I., *Critique of Pure Reason*, A598/B626]. \"God is omnipotent\" attributes a property; \"God exists\" does not attribute a property but rather asserts that the concept of God is instantiated.\n\nThis view is embedded in the standard logical treatment: in first-order predicate logic, existence is expressed by the existential quantifier (โˆƒx), not by a predicate. \"Tigers exist\" becomes \"there is at least one thing that is a tiger,\" not \"tigers have the property of existence.\"\n\nIf existence is not a predicate, the ontological argument for God's existence โ€” which treats existence as a perfection that maximally great beings must have โ€” fails at the point of treating existence as a property that can be possessed in greater or lesser degree.\n\n## Meinong and Non-Existent Objects\n\nAlexius Meinong argued that the domain of objects is wider than the domain of existents ^[Meinong, A., \"On the Theory of Objects\", 1904]. There are objects โ€” the golden mountain, the round square, Sherlock Holmes โ€” that do not exist. These non-existent objects nonetheless have properties: the golden mountain is golden and mountainous; the round square is both round and square (and therefore impossible); Sherlock Holmes lived at 221B Baker Street.\n\nMeinong's *Theory of Objects* distinguishes *existence* (the mode of being of concrete things), *subsistence* (the mode of being of abstract objects like numbers and propositions), and mere *Sosein* (having a nature, without any mode of being). Non-existent objects have Sosein without existence.\n\nRussell objected vigorously to this \"Meinongian jungle\" of non-existent objects, calling it \"a failure of that robust sense of reality which ought to be preserved even in the most abstract studies\" ^[Russell, B., \"Review of Meinong\", *Mind* 14, 1905, p.533]. His theory of definite descriptions was designed to handle sentences about non-existents without ontological commitment to them.\n\n## Russell's Theory of Descriptions\n\nRussell's theory of descriptions analyses sentences of the form \"The F is G\" as: there is exactly one F, and that F is G ^[Russell, B., \"On Denoting\", *Mind* 14, 1905, pp.479-493]. \"The present King of France is bald\" is false because there is no present King of France โ€” the uniqueness condition fails. We do not need to posit a non-existent King of France; we only need to recognise that the sentence has a false existential presupposition.\n\nThis \"logical paraphrase\" strategy โ€” finding analyses of problematic sentences that avoid commitment to suspect entities โ€” became a central tool of analytic philosophy. Ockham's razor (\"Do not multiply entities beyond necessity\") is the methodological principle: ontological economy is a theoretical virtue.\n\n## Contemporary Debates\n\n**Metaontology** โ€” the study of what ontological questions mean and how they should be answered โ€” has become a major area. Carnap distinguished *internal* questions (do numbers exist, within the mathematical framework?) from *external* questions (does the mathematical framework correspond to reality?) and argued that external questions are pragmatic, not factual ^[Carnap, R., \"Empiricism, Semantics and Ontology\", *Revue Internationale de Philosophie* 4, 1950]. Quine rejected this distinction; contemporary metaontologists debate whether it can be rehabilitated.\n\n**Ontological pluralism** โ€” the view that existence itself is not univocal, but that there are different \"modes of being\" โ€” has been defended by Kris McDaniel and others, reviving something like the Meinongian project with better tools ^[McDaniel, K., *The Fragmentation of Being*, Oxford UP, 2017].\n\n**Truthmaker theory** holds that truths are made true by features of reality, and asks what those features are for different classes of truths โ€” mathematical truths, modal truths, moral truths ^[Armstrong, D.M., *Truth and Truthmakers*, Cambridge UP, 2004].\n\nExistence and being may seem like the most abstract possible questions. But they have concrete implications: whether numbers exist bears on the foundations of mathematics; whether moral properties exist bears on the nature of moral knowledge; whether merely possible objects exist bears on modal semantics. Ontology is, as Quine put it, where logic and the world meet." + }, + { + "file": "pages/meta-02-identity.md", + "title": "Identity and Persistence", + "section-id": "metaphysics", + "keywords": "", + "description": "The Ship of Theseus, personal identity, and four-dimensionalist theories of persistence.", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Identity and Persistence\n\nWhat makes a thing the same thing over time? The ship that returned to Athens was rebuilt plank by plank during the voyage; no original material remained. Is it the same ship? The person who wakes tomorrow morning shares your memories and psychology โ€” but not your matter, since your cells replace themselves over years. Are they you?\n\nThese are not idle puzzles. They bear on personal survival, moral responsibility, and the metaphysics of change, and their answers connect to fundamental questions about what kinds of things exist.\n\n## The Ship of Theseus\n\nThe Ship of Theseus is one of philosophy's oldest thought experiments, recorded by Plutarch ^[Plutarch, *Theseus*, ch.23, c.75 CE]. Its philosophical interest lies not in the historical case but in the structure of the puzzle it reveals: ordinary objects persist through gradual material change, but taken to the limit, material continuity seems to dissolve.\n\nHobbes added a further twist: suppose someone keeps all the original planks and reassembles them. Which is the original ship โ€” the continuously maintained vessel or the reconstructed one? ^[Hobbes, T., *De Corpore*, 1655, II.xi.7]. The puzzle reveals that \"same ship\" may be determined by different identity criteria (material continuity, spatial-temporal continuity, functional continuity) that can diverge.\n\nThis connects to a broader metaphysical debate about *constitution*: when a lump of clay is shaped into a statue, are there two objects (the lump and the statue) that share matter but have different persistence conditions? Or only one? Defenders of constitution theory say two; critics say one (and find the view of two things in the same place implausible) ^[Wiggins, D., *Sameness and Substance*, Blackwell, 1980; Gibbard, A., \"Contingent Identity\", *Journal of Philosophical Logic* 4, 1975].\n\n## Personal Identity: The Classical View\n\nJohn Locke offered the first systematic philosophical analysis of personal identity ^[Locke, J., *Essay*, II.xxvii]. He distinguished the identity of *substance* (a material thing, continuous in matter), *organism* (a living thing, continuous in life), and *person* (a thinking conscious being, continuous in *consciousness*).\n\nPersons, for Locke, persist through *psychological continuity* โ€” specifically, memory. What makes the person who did action A the same person as me is my memory of having done A. This allows persons to come apart from both bodies and souls: the prince and the cobbler might \"swap\" if their consciousnesses were somehow exchanged.\n\n**Objections:** Bishop Butler accused Locke of circularity: memory presupposes personal identity, so it cannot constitute it ^[Butler, J., *The Analogy of Religion*, 1736, Appendix I]. Thomas Reid's *brave officer paradox* sharpened this: an old general remembers his younger self's bravery as a junior officer, but the officer (flogged as a boy) no longer remembers the boy's actions. By transitivity, the general is not the same person as the boy โ€” but this seems absurd ^[Reid, T., *Essays on the Intellectual Powers of Man*, 1785, III.6].\n\n## Neo-Lockean Theories\n\nDerek Parfit developed the most sophisticated neo-Lockean account ^[Parfit, D., *Reasons and Persons*, Oxford UP, 1984, Part III]. He replaced memory with *psychological continuity* โ€” overlapping chains of psychological connections (memories, intentions, beliefs, desires) โ€” and argued that what matters for survival is not strict identity but this continuity relation.\n\nParfit's striking conclusion: personal identity is not what matters in survival. In cases of fission (where your psychology is duplicated in two people), neither resulting person is strictly you โ€” but both have what matters as much as survival does. We should care about psychological continuity and connectedness, not about identity itself.\n\nThis has radical implications for ethics: if personal identity does not matter in itself, many concerns about future persons โ€” including concerns about one's own future self โ€” are impersonal, and the boundaries between persons may be less sharp than we normally assume.\n\n## The Biological Criterion\n\nEric Olson has argued for *animalism*: we are human animals, and our persistence conditions are those of biological organisms ^[Olson, E., *The Human Animal*, Oxford UP, 1997]. Personal identity is not constituted by psychological continuity; you persist as long as your body's metabolism continues. Brain transplants, on this view, are body transplants.\n\nAnimalism avoids neo-Lockean puzzles by refusing to split the person from the organism, but it faces difficulties with cerebral bisection, personal survival, and cases where psychological continuity intuitively tracks identity better than biological continuity does.\n\n## Four-Dimensionalism\n\nThe *four-dimensionalist* (or *perdurantist*) view holds that objects persist by having temporal parts at different times, just as they have spatial parts at different locations ^[Lewis, D., *On the Plurality of Worlds*, Blackwell, 1986, pp.202-204; Sider, T., *Four-Dimensionalism*, Oxford UP, 2001]. The person who existed yesterday is a *temporal part* of a four-dimensional object extended through time as well as space.\n\nOn this view, there is no problem of persistence through change: different temporal parts can have different properties. The Ship of Theseus problem dissolves: the ship at time t1 and the ship at time t2 are different temporal parts of the same four-dimensional whole, even though their material composition differs.\n\nFour-dimensionalism is technically elegant but counterintuitive: it implies that we never change โ€” what we ordinarily call \"my change\" is two different temporal parts having different properties. And the multiplication of temporal parts raises concerns about parsimony.\n\n**Three-dimensionalists** (or *endurantists*) hold that ordinary objects persist by being *wholly present* at each moment of their existence. They accept genuine identity through time and must therefore give an account of how the same thing can have different properties at different times (through *temporally modified* property ascription, or relativisation to times).\n\nThe persistence debate connects to the metaphysics of time: *presentists*, who hold only the present exists, are naturally endurantists; *eternalists*, who hold past, present, and future equally exist, can more naturally accommodate perdurance." + }, + { + "file": "pages/meta-03-causation.md", + "title": "Causation", + "section-id": "metaphysics", + "keywords": "", + "description": "Hume's regularity account of causation, counterfactual theories, mechanistic theories, and causal pluralism.", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Causation\n\nCausation is among the most pervasive features of reality and among the most philosophically contested. We invoke causal relations constantly: the window broke because it was struck by the ball; the fire spread because of the wind; the patient died because of the infection. Causal explanation, causal reasoning, and causal intervention underlie science, medicine, law, and everyday thought. Yet what causation *is* โ€” what it is for one event to cause another โ€” remains deeply controversial.\n\n## Hume's Regularity Theory\n\nHume's analysis of causation, discussed in the context of empiricism (Chapter 4), remains the starting point for the contemporary debate. Hume distinguished two definitions of cause.\n\nThe first, in terms of *constant conjunction*: \"An object, followed by another, and where all the objects similar to the first are followed by objects similar to the second\" ^[Hume, D., *Enquiry*, ยง7.2]. Causation, on this view, is nothing over and above regular succession: whenever an event of type A occurs, an event of type B follows.\n\nThe second, in terms of *determination*: \"An object followed by another, and whose appearance always conveys the thought to that other.\" This psychological definition reveals that the impression of necessary connection is in us, not in the objects.\n\nThe *regularity theory* developed from Hume's first definition. Mill systematised it with his *methods of agreement, difference*, and *concomitant variation* for identifying causal regularities ^[Mill, J.S., *A System of Logic*, 1843, III.viii].\n\n**Objections:** Mere regular succession does not suffice for causation. Day regularly precedes night, but dawn does not cause dusk. Common causes produce correlated effects โ€” thunder correlates with lightning, but neither causes the other. And regularities can be accidental (all gold spheres are smaller than the sun) rather than causal.\n\n## Counterfactual Theories\n\nDavid Lewis's *counterfactual theory* defines causation in terms of counterfactual dependence: C causes E if and only if, had C not occurred, E would not have occurred ^[Lewis, D., \"Causation\", *Journal of Philosophy* 70, 1973, pp.556-567].\n\nThis approach handles many cases better than regularity theories. The counterfactual \"if the ball had not struck the window, the window would not have broken\" is true (under normal conditions); hence the striking caused the breaking. Cases of accidental correlation are handled naturally: even if thunder and lightning are regularly correlated, it is not true that if thunder had not occurred, lightning would not have.\n\n**Problems:** *Preemption* โ€” where two potential causes compete and one \"preempts\" the other โ€” is difficult. If two assassins shoot simultaneously and one bullet arrives first, the first shot caused the death; but it is not clear that the death counterfactually depends on the first shot, since the second would have caused it anyway. Lewis developed increasingly complex responses involving *fragility*, *quasi-dependence*, and *influence* ^[Lewis, D., \"Causation as Influence\", *Journal of Philosophy* 97, 2000].\n\n*Overdetermination* โ€” where two simultaneous causes each suffice for the effect โ€” creates parallel problems.\n\n## Mechanistic Theories\n\n*Mechanistic* or *process* theories hold that causation consists in the transmission of energy, momentum, or causal influence through a spatiotemporally continuous process ^[Salmon, W., *Scientific Explanation and the Causal Structure of the World*, Princeton UP, 1984; Dowe, P., *Physical Causation*, Cambridge UP, 2000].\n\nWesley Salmon proposed that causal processes are distinguished from *pseudo-processes* (like shadows) by their ability to transmit a *mark* โ€” an alteration made at one point that propagates forward. Phil Dowe replaced this with a conserved quantity account: a causal process is one that transmits a conserved quantity (energy, charge, momentum).\n\nMechanistic theories have the advantage of closely tracking scientific practice โ€” physicists and biologists routinely explain by identifying mechanisms. But they face difficulties with causation by absence (the bridge collapsed *because* the engineers failed to inspect it), negative causation, and the causation of absences.\n\n## Interventionist Theories\n\nJames Woodward developed an *interventionist* account, appealing to the notion of ideal intervention ^[Woodward, J., *Making Things Happen*, Oxford UP, 2003]. A variable X causes Y if there is a possible ideal intervention on X (one that changes X independently of other causes of Y) that changes Y.\n\nThis connects causation to the notion of manipulation or control, and is particularly well-suited to the social and biological sciences. It captures the idea that causal claims are action-guiding: to know that X causes Y is to know that intervening on X will change Y.\n\nInterventionism faces the question of whether the interventionist account is circular โ€” since interventions are themselves causal notions.\n\n## Singular Causation and the Problem of Many Levels\n\nA recurring question: is causation a relation between *types* of events (event-type A regularly precedes event-type B) or between *tokens* โ€” particular, individual events (this striking caused this breaking)?\n\nToken causation matters for law: we want to know whether *this* person's negligence caused *this* accident, not whether negligence-type events generally precede accidents.\n\nThe *problem of causal exclusion* (closely connected to philosophy of mind) asks how mental causation is possible if everything is determined at the physical level. If the physical causes of my action fully determine it, what work is left for my mental states to do? ^[Kim, J., *Mind in a Physical World*, MIT Press, 1998]. This challenges non-reductive physicalism about mind.\n\n**Causal pluralism** holds that there is no single analysis of causation that captures all uses of causal vocabulary ^[Hall, N., \"Two Concepts of Causation\", in *Causation and Counterfactuals*, ed. Collins et al., MIT Press, 2004]. There may be one concept of causation for physics, another for biology, another for the law. Pluralism is comfortable with this; the search for a unified account may be misconceived.\n\nThe contemporary causation debate is technically sophisticated and connects to philosophy of science, philosophy of mind, and action theory. What is clear is that Hume was right about one thing: the concept of causation is not simply read off the surface of experience but requires serious philosophical analysis." + }, + { + "file": "pages/meta-04-freewill.md", + "title": "Free Will and Determinism", + "section-id": "metaphysics", + "keywords": "", + "description": "Hard determinism, libertarianism, compatibilism, and Frankfurt cases.", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Free Will and Determinism\n\nThe free will debate is among philosophy's most enduring and personally consequential. It asks whether, in a deterministic universe, human beings can be genuinely free โ€” free in a way that makes praise, blame, punishment, and moral responsibility appropriate. The question matters not just theoretically but practically: legal systems, personal relationships, and our self-understanding all presuppose that people can be held responsible for their actions.\n\n## The Incompatibilist Intuition\n\nThe *basic argument* for incompatibilism runs roughly as follows ^[Van Inwagen, P., *An Essay on Free Will*, Oxford UP, 1983, pp.56-105]:\n\n1. Determinism is true: every event, including every human action, is causally necessitated by prior events in conjunction with the laws of nature.\n2. If determinism is true, then no one ever could have acted otherwise than they did.\n3. Moral responsibility requires the ability to have acted otherwise.\n4. Therefore, if determinism is true, no one is morally responsible for anything.\n\nThis argument has considerable intuitive force. If my action was causally determined by events that happened before I was born, in what sense was it *my* choice? If the complete causal history of the universe made my decision inevitable, how am I the author of it in any meaningful sense?\n\n## Hard Determinism\n\n*Hard determinists* accept incompatibilism and accept determinism, concluding that free will does not exist and moral responsibility must be radically revised or abandoned.\n\nDerk Pereboom has argued for *hard incompatibilism* with increasing sophistication ^[Pereboom, D., *Living Without Free Will*, Cambridge UP, 2001]. He accepts that moral luck undermines responsibility, that determinism (or indeterminism, for different reasons) threatens desert-based punishment, but argues that a meaningful life can be constructed without the reactive attitudes (blame, indignation, gratitude) that presuppose responsibility.\n\nThe practical implications are significant: criminal punishment on retributive grounds is unjustified; therapeutic and quarantine-based reasons for incapacitation remain. Reactive attitudes would be gradually replaced by more forward-looking responses.\n\n## Libertarianism About Free Will\n\n*Libertarians* (in the metaphysical sense, entirely distinct from political libertarianism) accept incompatibilism but reject determinism, maintaining that free will requires โ€” and we have โ€” a form of causation that is not deterministic.\n\nAgent causation: Roderick Chisholm argued that free action requires *agent causation* โ€” a primitive, irreducible capacity of persons as agents to initiate causal chains, not wholly determined by prior events ^[Chisholm, R., \"Human Freedom and the Self\", in *Free Will*, ed. Watson, Oxford UP, 1982].\n\nThe *undetermined choice*: Robert Kane developed an account on which free will-exercising decisions occur at moments of *self-forming actions* โ€” quantum-indeterminate moments of genuine undeterminedness, where the agent's character and reasons could have produced either outcome ^[Kane, R., *The Significance of Free Will*, Oxford UP, 1996].\n\nLibertarianism faces the *luck objection*: if my action was undetermined, then it seems random rather than free. An undetermined choice is one that even I could not have predicted from my own character and reasons โ€” which seems to undermine rather than secure my authorship of the action.\n\n## Compatibilism\n\n*Compatibilists* reject the third premise of the basic argument. Moral responsibility, they argue, does not require the ability to have done otherwise in the libertarian sense. What matters is whether the action was performed for the right kinds of reasons, whether the agent was responsive to reasons, whether the action was voluntary in the relevant sense.\n\n**Classical compatibilism:** Hume, and following him most analytic philosophers of the twentieth century, held that freedom is simply the ability to act in accordance with one's own desires, without external compulsion ^[Hume, D., *Enquiry*, ยง8]. Coercion, addiction, and phobia compromise freedom; determinism does not.\n\n**Hierarchical compatibilism:** Harry Frankfurt proposed that what matters for freedom is the *structure* of the agent's motivations ^[Frankfurt, H., \"Freedom of the Will and the Concept of a Person\", *Journal of Philosophy* 68, 1971, pp.5-20]. We have first-order desires (I want to smoke) and second-order desires (I want to want to smoke, or I want not to want to smoke). A free agent is one whose first-order desires align with their second-order volitions โ€” who acts from desires they endorse. This hierarchical structure distinguishes the wanton (who acts on whatever desire is strongest) from the autonomous agent.\n\n**Reasons-responsiveness:** John Martin Fischer and Mark Ravizza developed the view that free agency requires mechanisms that are *reasons-responsive* โ€” mechanisms that would have produced different choices had there been different reasons ^[Fischer, J.M. and Ravizza, M., *Responsibility and Control*, Cambridge UP, 1998]. You are responsible for actions that flow from your own reasons-responsive mechanisms.\n\n## Frankfurt Cases\n\nHarry Frankfurt's most influential contribution is the *Frankfurt case*, designed to challenge the *Principle of Alternative Possibilities* (PAP): a person is morally responsible for their action only if they could have acted otherwise.\n\nThe structure: Black wants Jones to perform some action. Black has installed a mechanism in Jones's brain that will, if Jones shows any sign of not performing the action, intervene and ensure Jones performs it. In fact, Jones performs the action on his own, and the mechanism never activates. Jones could not have done otherwise (the mechanism would have prevented it). But, Frankfurt argues, Jones is clearly morally responsible ^[Frankfurt, H., \"Alternate Possibilities and Moral Responsibility\", *Journal of Philosophy* 66, 1969, pp.829-839].\n\nIf Frankfurt cases are sound, PAP is false, and the basic incompatibilist argument loses its third premise. The literature on Frankfurt cases is enormous: compatibilists have used them to argue that alternative possibilities are not required for responsibility; incompatibilists have argued that Frankfurt cases either fail to establish genuine alternative possibilities being closed off, or leave open a \"flicker of freedom\" ^[Kane, R., \"Two Kinds of Incompatibilism\", *Philosophy and Phenomenological Research* 50, 1989].\n\n## Strawson's Reactive Attitudes\n\nP.F. Strawson argued that the debate misses what is most important about responsibility: our *reactive attitudes* ^[Strawson, P.F., \"Freedom and Resentment\", *Proceedings of the British Academy* 48, 1962]. Resentment, gratitude, indignation, and love are the appropriate responses to the quality of an agent's will toward us. These attitudes constitute โ€” rather than presuppose โ€” our moral practices. The question of whether determinism is true is largely beside the point; what matters is whether we are the kinds of creatures to whom it is appropriate to hold reactive attitudes, and we clearly are.\n\nStrawson's insight is that moral responsibility is essentially an interpersonal, practice-constituted phenomenon, not primarily a metaphysical one.\n\nThe free will debate has not converged. Compatibilism is the most widely held view among professional philosophers, but libertarian and hard incompatibilist positions retain significant defenders. The question of what freedom requires โ€” and whether we have it โ€” remains genuinely open." + }, + { + "file": "pages/meta-05-mind.md", + "title": "Philosophy of Mind", + "section-id": "metaphysics", + "keywords": "", + "description": "Dualism, functionalism, physicalism, qualia, and the hard problem of consciousness.", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Philosophy of Mind\n\nPhilosophy of mind asks what the mind is, how mental states relate to physical states, and whether consciousness can be explained by the natural sciences. It is a meeting point of metaphysics, epistemology, cognitive science, and neuroscience โ€” and at its centre lies what David Chalmers called *the hard problem of consciousness*, the question of why there is subjective experience at all.\n\n## Substance Dualism\n\nDescartes' *substance dualism* holds that mind and body are distinct substances: the body is extended substance (res extensa), governed by mechanical laws; the mind is thinking substance (res cogitans), unextended and not subject to physical laws ^[Descartes, R., *Meditations*, AT VII:78-80; *The Passions of the Soul*, AT XI:330].\n\nSubstance dualism captures the intuition that mental life โ€” the experience of pain, the feeling of red, the taste of coffee โ€” is radically different in kind from the physical world. No description in purely physical terms seems to capture what it is like to be in pain.\n\nThe central objection: *causal interaction*. If mind and body are distinct substances, how do they causally interact? How does my decision to raise my arm cause my arm to rise? Descartes' attempted answer โ€” via the pineal gland โ€” was never convincing. Occasionalism (Malebranche) and pre-established harmony (Leibniz) were developed as alternatives, both of which deny genuine causal interaction and invoke God.\n\n## Behaviourism and Its Failures\n\n*Logical behaviourism* โ€” associated with Ryle and early Wittgenstein โ€” held that mental concepts are analysable in terms of behavioural dispositions, not inner states ^[Ryle, G., *The Concept of Mind*, 1949]. To believe that it will rain is to be disposed to carry an umbrella, to seek shelter, and so on. There is no \"ghost in the machine\" โ€” mentality just is the complex of behavioural dispositions.\n\nHilary Putnam argued that behaviourism fails because behavioural dispositions are mediated by other mental states ^[Putnam, H., \"Brains and Behavior\", 1963]. The pain-disposition to withdraw from stimuli requires the desire to avoid pain; the belief-disposition to seek shelter requires the desire to stay dry. No purely behavioural analysis of a mental state can avoid this regress.\n\n## Identity Theory\n\n*Identity theory* โ€” associated with Place and Smart โ€” held that mental states are identical to brain states: pain is a type of neural activity ^[Place, U.T., \"Is Consciousness a Brain Process?\", *British Journal of Psychology* 47, 1956; Smart, J.J.C., \"Sensations and Brain Processes\", *Philosophical Review* 68, 1959].\n\n**Multiple realisability objection:** Putnam argued that mental states are *multiply realisable* โ€” they can be realised in many different physical substrates ^[Putnam, H., \"Psychological Predicates\", 1967]. If octopuses (with radically different nervous systems) can be in pain, pain cannot be identical to any specific neural state. This counts against *type* identity theory, though not *token* identity (this instance of pain is identical to this neural event).\n\n## Functionalism\n\n*Functionalism* โ€” Putnam's alternative โ€” holds that mental states are defined by their *functional role*: their causal relations to sensory inputs, behavioural outputs, and other mental states ^[Putnam, H., \"The Nature of Mental States\", 1967]. Pain is whatever state is typically caused by tissue damage, causes withdrawal behaviour and the desire to relieve it, and interacts with other states in characteristic ways.\n\nFunctionalism accommodates multiple realisability: what makes something a pain is its functional role, regardless of whether it is implemented in neurons, silicon, or anything else. It is the dominant view in philosophy of mind and cognitive science.\n\n**Objections:** The *inverted qualia* argument: two people might have their functional roles entirely aligned while having inverted phenomenal experiences (what's red to you is green to me). Functionalism, which is defined by function, cannot distinguish them. The *absent qualia* argument: a system could have all the right functional relations while having no phenomenal experience at all โ€” a philosophical zombie ^[Block, N., \"Troubles with Functionalism\", *Minnesota Studies in the Philosophy of Science* 9, 1978; Chalmers, D., *The Conscious Mind*, 1996, ch.3].\n\n## Physicalism and Its Varieties\n\nContemporary philosophy of mind is broadly physicalist: mental states are physical states or at least entirely dependent on physical states. The question is *how* they are dependent.\n\n*Supervenience physicalism*: mental properties supervene on physical properties โ€” any two individuals physically identical are mentally identical ^[Kim, J., *Supervenience and Mind*, Cambridge UP, 1993].\n\n*Non-reductive physicalism*: mental properties are real but not reducible to physical properties, even though they supervene on them.\n\n*Reductive physicalism*: mental properties can ultimately be explained in physical terms.\n\nThe *causal exclusion argument* (Kim) poses a serious problem for non-reductive physicalism: if physical events have sufficient physical causes, and mental events are supposed to cause behaviour, then either mental events are physical events (reductivism) or mental events are causally redundant ^[Kim, J., *Mind in a Physical World*, 1998, ch.2-3].\n\n## The Hard Problem\n\nChalmers distinguished the *easy problems* of consciousness โ€” explaining cognitive access, attention, introspection, sleep/waking cycles (these problems are hard, but they admit of functional-explanatory solutions) โ€” from *the hard problem*: why is there subjective experience at all? ^[Chalmers, D., \"Facing Up to the Problem of Consciousness\", *Journal of Consciousness Studies* 2, 1995].\n\nEven a complete physical and functional account of what the brain does would leave open why any of this processing is *experienced* โ€” why it feels like something to be a brain. The explanatory gap between physical descriptions and phenomenal experience seems irreducible.\n\nResponses range from *type-B physicalism* (the gap is a conceptual illusion, not a real explanatory gap) to *property dualism* (phenomenal properties are real, non-physical properties that supervene on physical ones) to *panpsychism* (consciousness is a fundamental feature of reality, found at all levels of physical organisation) ^[Goff, P., *Galileo's Error*, 2019].\n\nThe hard problem has not been solved. Whether it is solvable within a physicalist framework, or whether it requires revising our fundamental ontology, remains one of philosophy's most contested open questions." + }, + { + "file": "pages/meta-06-time.md", + "title": "The Nature of Time", + "section-id": "metaphysics", + "keywords": "", + "description": "A-series and B-series, presentism, eternalism, and the growing block theory of time.", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# The Nature of Time\n\nTime is both utterly familiar and deeply puzzling. We live in it, measure it, experience its passage. Yet when we ask what time is, whether the past and future exist, whether time flows or merely seems to, we find ourselves quickly in some of philosophy's most difficult territory.\n\n## McTaggart's A-Series and B-Series\n\nJ.M.E. McTaggart introduced the most influential framework for the philosophy of time in his 1908 paper \"The Unreality of Time\" ^[McTaggart, J.M.E., \"The Unreality of Time\", *Mind* 17, 1908, pp.457-474].\n\nThe *B-series* is the ordering of events as earlier, simultaneous, and later. Every event stands in a fixed B-relation to every other: the Battle of Hastings is earlier than the French Revolution; your birth is earlier than your reading this sentence. These relations are *permanent*: if A is earlier than B, it always has been and always will be.\n\nThe *A-series* orders events as *past*, *present*, and *future*. Unlike B-relations, A-properties change: what is now future becomes present and then past. The event of your reading this sentence was future, is now present, and will be past.\n\nMcTaggart argued that the A-series is essential to time โ€” without the genuine distinction between past, present, and future, there would be no temporal becoming, no flow of time, and time would not be genuinely real. But the A-series is contradictory: every event has all three properties (past, present, future), which are mutually exclusive. Attempts to resolve the contradiction by saying \"the event is present *now*, past *at later times*, future *at earlier times*\" invoke further temporal moments and generate a vicious infinite regress. McTaggart concluded that time is unreal.\n\nMost philosophers reject the conclusion but accept that McTaggart identified a genuine structural puzzle.\n\n## Presentism\n\n*Presentism* holds that only present entities exist ^[Crisp, T., \"Presentism\", in *Oxford Handbook of Metaphysics*, 2003]. The past is gone; the future is not yet here. \"There were dinosaurs\" is true, but dinosaurs do not now exist in any sense โ€” they existed and no longer do.\n\nPresentism is the view that most naturally fits everyday temporal experience. The past seems gone; the future seems open.\n\n**The problem of cross-temporal relations:** Many true claims seem to relate present entities to past ones: Caesar crossed the Rubicon *before* you were born. If Caesar does not now exist, what makes this claim true? Presentists have responded with *truthmakers* that exist presently โ€” facts about the present state of the world โ€” and with *temporal ersatzism* (abstract representations of past times).\n\n**Reconciling presentism with special relativity:** Relativity implies there is no absolute simultaneity โ€” different reference frames carve the four-dimensional spacetime differently. This is deeply problematic for presentism, which requires a distinguished present ^[Putnam, H., \"Time and Physical Geometry\", *Journal of Philosophy* 64, 1967].\n\n## Eternalism (The Block Universe)\n\n*Eternalism* โ€” the view associated with Einstein's spacetime โ€” holds that past, present, and future entities all equally exist, merely at different temporal locations ^[Sider, T., *Four-Dimensionalism*, Oxford UP, 2001, ch.2]. The universe is a four-dimensional \"block\" in which all events co-exist; the distinction between past, present, and future is merely perspectival, like the distinction between here and there.\n\nEternalism accommodates special relativity naturally: there is no privileged present, and the temporal order of events can be frame-relative.\n\n**The problem of temporal passage:** Eternalism seems to make temporal experience mysterious. If past and future are equally real, why does time seem to pass? Why do we experience becoming rather than merely co-existing with our past and future selves in a static four-dimensional block?\n\nSome eternalists accept this implication and argue that the *passage* of time is an illusion โ€” a product of our temporal perspective, not a feature of reality. Others have argued that passage can be accommodated within an eternalist framework through the *moving spotlight* theory.\n\n## The Growing Block Theory\n\nC.D. Broad proposed an intermediate view: the growing block ^[Broad, C.D., *Scientific Thought*, 1923, pp.66-68]. The past and present are real and fixed; the future does not yet exist. As time passes, new events come into existence and the block grows. This preserves the reality of temporal becoming without committing to a privileged present.\n\n**Objections:** If the growing block is correct, and past moments are real and fixed, how do we know we are in the present rather than in some past moment (which is, after all, equally real)? This generates a peculiar form of scepticism about our temporal location ^[Tooley, M., *Time, Tense, and Causation*, Oxford UP, 1997].\n\n## The Direction of Time\n\nPhysics, at the fundamental level, is largely time-symmetric: the laws of mechanics, electromagnetism, and quantum mechanics (with minor exceptions) work equally in both temporal directions. Yet our experience of time is strongly asymmetric: we remember the past and not the future; causes precede effects; entropy increases.\n\nBoltzmann argued that the thermodynamic arrow of time โ€” the increase of entropy โ€” explains the asymmetry ^[Boltzmann, L., *Lectures on Gas Theory*, 1896-98]. We are in a low-entropy region of a vastly larger, mostly high-entropy universe; the Second Law of Thermodynamics is a statistical fact about macroscopic systems, not a fundamental law.\n\nThe *causal theory* of time holds that the direction of time is grounded in the direction of causation: the future is the direction in which causal processes flow. But if causation itself is time-asymmetric, this seems circular.\n\n**Temporal experience:** Husserl's phenomenology of time-consciousness distinguished *retention* (the immediate just-past), *primal impression* (the now), and *protention* (the immediate just-future) as three interlocking structures of temporal awareness ^[Husserl, E., *On the Phenomenology of the Consciousness of Internal Time*, 1928]. The experience of time as flowing may be constituted by this structure rather than being evidence of metaphysical flow.\n\nThe nature of time remains one of the most contested and most fundamental questions in metaphysics, connecting to physics, the theory of causation, personal identity, and the phenomenology of experience." + }, + { + "file": "pages/preface.md", + "title": "Preface", + "section-id": "front-matter", + "keywords": "", + "description": "Professor Okafor's preface โ€” why philosophy matters, how to use this book, and acknowledgements.", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "![Library and scholarly atmosphere](assets/images/hero.jpg)\n\n# Preface\n\nPhilosophy begins where certainty ends. It is the sustained, rigorous attempt to think carefully about questions that do not yield to experiment, calculation, or common sense alone โ€” questions about the nature of knowledge, the structure of reality, the foundations of morality, and the meaning of a human life. These questions are not trivial or marginal. They are, in many respects, the questions that underlie everything else we do. The scientist who believes her experiments can yield knowledge presupposes an account of knowledge and justification. The judge who sentences a criminal presupposes an account of moral responsibility and desert. The citizen who argues for a just social arrangement presupposes an account of fairness, rights, and legitimate authority.\n\nPhilosophy is the discipline that examines these presuppositions โ€” that takes them seriously enough to interrogate them rather than quietly relying on them.\n\nThis book is an introduction to three of philosophy's central subdisciplines: epistemology (the theory of knowledge), metaphysics (the study of the fundamental nature of reality), and ethics (the systematic examination of morality). These three areas are not the whole of philosophy โ€” there is also philosophy of language, philosophy of science, political philosophy in its own right, philosophy of mathematics, philosophy of religion, and many others โ€” but they constitute what has traditionally been called the core of the discipline, and they are deeply interconnected. Our account of knowledge bears on our account of what exists. Our account of what exists bears on our account of moral facts. The connections run in every direction.\n\n## On This Book\n\n*Foundations of Modern Philosophy* is written for advanced undergraduates and first-year graduate students who have some acquaintance with philosophical questions but no prior systematic training. It assumes intellectual seriousness but not technical background. The goal is not to survey every debate in every area โ€” that would require a library, not a textbook โ€” but to provide rigorous introductions to the central issues, equip the reader with the conceptual vocabulary needed to engage with primary sources, and convey something of the excitement of philosophy as a living intellectual enterprise.\n\nEach chapter introduces a major area of inquiry, presents the central arguments with the care they deserve, and points toward the ongoing debates that the reader will encounter if they pursue the subject further. The book is designed to be read sequentially, but chapters can also be read independently: those with a specific interest in, say, philosophy of mind or free will can begin there and follow the cross-references.\n\nI have tried to write with clarity without sacrificing rigour, and with genuine intellectual engagement without pretending that the questions have been settled. They have not been settled. Some of them may not be settleable. This is not a reason for despair but for sustained attention.\n\n## Philosophy and Disagreement\n\nOne thing that surprises new students of philosophy is how much disagreement persists among careful, intelligent, well-informed people. In mathematics, experts eventually converge. In philosophy, they often do not. This is sometimes taken as evidence that philosophy makes no progress, or that philosophical questions are somehow empty.\n\nI believe this is wrong, for two reasons.\n\nFirst, philosophical progress is real, even when consensus is absent. We understand the problems more precisely than we did. The conceptual terrain has been mapped. Certain paths have been shown to be dead ends. The space of viable positions has been narrowed, even if it has not collapsed to a single point.\n\nSecond, sustained intelligent disagreement is not a failure โ€” it is what happens when the questions are genuinely hard and the standards of evidence genuinely exacting. The fact that Kant, Mill, and Aristotle disagree about the foundations of morality does not mean there are no good arguments in ethics. It means the questions are difficult enough that even great minds approach them from different directions and reach different conclusions.\n\nStudents who enter philosophy looking for certainty will be disappointed. Students who enter looking for difficult questions asked well, with intellectual honesty and genuine rigour, will find that and more.\n\n## Acknowledgements\n\nThis book has accumulated debts over many years of teaching. Students at the University of Lagos, Cambridge, and Princeton asked the questions that forced me to think more carefully; colleagues in seminars and conference rooms challenged positions I held too comfortably; many generations of undergraduates reminded me, by their confusion and their insight alike, what it is actually like to encounter these ideas for the first time.\n\nI am grateful to my research assistants, Amara Osei-Bonsu and Lars Eriksson, for careful reading of the manuscript. My editor, who has the good philosopher's gift of asking exactly the right question at exactly the wrong moment, improved the book considerably.\n\nMy deepest debts are to the philosophers whose work is discussed in these pages. I have tried to present their arguments with fairness. Where I have failed, the failure is mine.\n\n*James Okafor*\n*Lagos / Princeton, 2026*" + }, + { + "file": "pages/synthesis.md", + "title": "Synthesis and Open Questions", + "section-id": "conclusion", + "keywords": "", + "description": "How epistemology, metaphysics, and ethics connect, and ten open problems that define the frontiers of contemporary philosophy.", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Synthesis and Open Questions\n\nWe have traversed three great branches of philosophy โ€” epistemology, metaphysics, and ethics โ€” as if they were distinct territories. They are not. In this concluding chapter we trace some of the connections between them, then identify ten open problems that represent the active frontiers of contemporary philosophical inquiry.\n\n## How the Three Branches Connect\n\n### Epistemology and Metaphysics\n\nEpistemology and metaphysics are entangled at their foundations. The question \"What is there?\" (ontology) is inseparable from \"How can we know what there is?\" (epistemology). Kant made this explicit: our knowledge of reality is always knowledge of reality as structured by our cognitive apparatus. The thing-in-itself โ€” the world independent of our categories โ€” is unknowable.\n\nThe debate between realism and anti-realism in metaphysics has a direct epistemological dimension: scientific realists hold that our best theories give us genuine knowledge of the unobservable structure of reality; anti-realists (van Fraassen's constructive empiricism) hold that empirical adequacy, not truth, is the goal of science.\n\nScepticism is both an epistemological and a metaphysical thesis: if we cannot rule out the brain-in-a-vat hypothesis, then our beliefs about the external world may be systematically false. Responding to scepticism requires both an epistemological account of what knowledge requires and a metaphysical account of what makes beliefs true.\n\n### Metaphysics and Ethics\n\nFree will and determinism (Chapter 9) is the clearest intersection of metaphysics and ethics. Moral responsibility โ€” the foundation of our entire ethical and legal practice โ€” presupposes that agents could have done otherwise. If determinism is true (and it may be), then whether this condition is satisfied becomes a metaphysical question with profound ethical and social consequences.\n\nPersonal identity (Chapter 7) connects to ethics in multiple ways. Derek Parfit argued that the correct view of personal identity โ€” that what matters in survival is not identity per se but psychological continuity โ€” has far-reaching implications for distributive justice, self-interest, and our concern for future persons. If I am only loosely connected to my future self, do I have the same reasons for prudence?\n\n### Epistemology and Ethics\n\nMoral epistemology asks whether we can have knowledge of ethical truths, and if so, how. Empiricists about ethics hold that moral judgements are answerable to experience; rationalists hold that some moral truths are knowable a priori. The methodology of ethics โ€” intuitions as data, reflective equilibrium, thought experiments โ€” is itself a set of epistemological commitments.\n\nThe is-ought gap (Hume) is an epistemological constraint on ethical argument: purely factual premises cannot entail normative conclusions. This shapes what counts as a valid argument in ethics.\n\n## Ten Open Problems in Philosophy\n\n### 1. The Hard Problem of Consciousness\nWhy is there subjective experience at all? Why does processing information feel like anything? David Chalmers's formulation of the hard problem remains without consensus resolution. Physicalist accounts explain the functional and structural properties of mind but seem to leave out the *what it is like*.\n\n### 2. The Nature of Mathematical Objects\nAre mathematical objects abstract entities that exist independently of minds (Platonism), or are they mental constructs (constructivism), or merely useful fictions (fictionalism)? The unreasonable effectiveness of mathematics โ€” its uncanny applicability to physical reality โ€” demands explanation on any view.\n\n### 3. The Reference of Natural Kind Terms\nKripke and Putnam argued that natural kind terms (\"water,\" \"gold\") refer rigidly to their physical essences, not to descriptive clusters. But what determines reference? The causal-historical picture has unresolved problems for highly theoretical kinds (fields, virtual particles) and social/biological categories.\n\n### 4. The Status of Modality\nWhat grounds claims about necessity and possibility? Are possible worlds Lewisian concrete universes, abstract maximally consistent sets of propositions, or something else? The ontological costs of modal realism seem high; the alternatives face their own problems.\n\n### 5. Personal Identity Over Time\nDespite extensive philosophical work, we lack a fully satisfactory account of what makes you the same person you were ten years ago โ€” and what this should matter for ethics. Parfit's reductionism remains controversial.\n\n### 6. The Foundations of Probability\nThe three major interpretations โ€” frequentist, Bayesian (subjective), and propensity โ€” each face serious objections. The role of probability in quantum mechanics compounds the difficulty: are quantum probabilities objective features of reality or epistemic representations of our uncertainty?\n\n### 7. The Demarcation Problem\nWhat distinguishes science from non-science, or good science from pseudo-science? Popper's falsifiability criterion is widely acknowledged to be insufficient. String theory and cosmological multiverse theories generate empirical predictions only under contested conditions. Philosophy of science lacks an agreed criterion.\n\n### 8. Moral Realism and Evolutionary Debunking\nSharon Street's evolutionary debunking argument: if our moral faculties were shaped by natural selection for fitness rather than moral truth, then we have no reason to trust that our moral intuitions track mind-independent moral facts.^[Street, S. (2006). \"A Darwinian Dilemma for Realist Theories of Value.\" *Philosophical Studies*, 127(1).] Moral realists must either deflect this argument or explain why adaptive pressure would align our intuitions with moral truth.\n\n### 9. The Epistemology of Disagreement\nWhen two equally well-informed, well-reasoned individuals reach opposing conclusions, what should each do? The *conciliationist* view says each should move towards the other; the *steadfast* view says you can maintain your view if you have independent reasons. The answer has implications for political philosophy, scientific consensus, and religious belief.\n\n### 10. The Grounds of Normativity\nWhy does reason bind us? Kant's answer โ€” that rational nature is the source of all value โ€” faces both metaphysical challenges (what is rational nature?) and sceptical challenges (why should I care about what reason prescribes?). Korsgaard's attempt to ground normativity in reflective self-endorsement has been contested. This may be the most fundamental question in all of philosophy.\n\n## Conclusion\n\nPhilosophy does not progress by solving problems and discarding them. The questions examined in this book โ€” about knowledge, reality, and value โ€” are perennial because they arise from the structure of human thought itself. What philosophy offers is not definitive answers but greater clarity about what the questions are, greater rigour in evaluating proposed answers, and greater sensitivity to the hidden assumptions that shape all inquiry.\n\nThe student who completes this introduction will find these questions following them into every discipline they pursue โ€” into science, into law, into medicine, into politics, into ordinary life. That is not a failure of philosophy to resolve itself. It is philosophy doing exactly what it should.\n\n## Further Reading\n\n- Chalmers, D. (2023). *Reality+: Virtual Worlds and the Problems of Philosophy*. Penguin.\n- Parfit, D. (1984). *Reasons and Persons*. Oxford University Press.\n- Nagel, T. (1986). *The View from Nowhere*. Oxford University Press." + } +] \ No newline at end of file diff --git a/sample-sites/modern-philosophy/theme.yml b/sample-sites/modern-philosophy/theme.yml new file mode 100644 index 0000000..0f52faf --- /dev/null +++ b/sample-sites/modern-philosophy/theme.yml @@ -0,0 +1,46 @@ +# mdcms theme โ€” Foundations of Modern Philosophy +light: + accent: "#744210" + background: "#FFFFF8" + nav-background: "#F7F3E9" + text: "#1A202C" + text-muted: "#718096" + +dark: + accent: "#F6AD55" + background: "#1A1611" + nav-background: "#211D18" + text: "#EDF2F7" + text-muted: "#A0AEC0" + +colours-semantic: + info: "#2B6CB0" + warning: "#C05621" + success: "#276749" + error: "#C53030" + +callouts: + info: + icon: info + primary-colour: "#2B6CB0" + background-colour: "#2B6CB0" + warning: + icon: warning + primary-colour: "#C05621" + background-colour: "#C05621" + success: + icon: success + primary-colour: "#276749" + background-colour: "#276749" + error: + icon: error + primary-colour: "#C53030" + background-colour: "#C53030" + +font-body: "bunny:Source Serif 4:400" +font-heading: "bunny:Source Serif 4:700" +font-size: 1.0 +line-height: 1.85 + +main-width: 72em +nav-width: 20em diff --git a/sample-sites/neuraldb-docs/assets/images/architecture.jpg b/sample-sites/neuraldb-docs/assets/images/architecture.jpg new file mode 100644 index 0000000..c16728a Binary files /dev/null and b/sample-sites/neuraldb-docs/assets/images/architecture.jpg differ diff --git a/sample-sites/neuraldb-docs/assets/images/dashboard.jpg b/sample-sites/neuraldb-docs/assets/images/dashboard.jpg new file mode 100644 index 0000000..c20e2f7 Binary files /dev/null and b/sample-sites/neuraldb-docs/assets/images/dashboard.jpg differ diff --git a/sample-sites/neuraldb-docs/assets/images/hero.jpg b/sample-sites/neuraldb-docs/assets/images/hero.jpg new file mode 100644 index 0000000..be27ec9 Binary files /dev/null and b/sample-sites/neuraldb-docs/assets/images/hero.jpg differ diff --git a/sample-sites/neuraldb-docs/config.yml b/sample-sites/neuraldb-docs/config.yml new file mode 100644 index 0000000..ef5113c --- /dev/null +++ b/sample-sites/neuraldb-docs/config.yml @@ -0,0 +1,6 @@ +# mdcms v0.3 | DO NOT REMOVE THIS COMMENT +sitename: NeuralDB +sitedescription: AI-native database with vector and relational capabilities +navigation: topbar +search: true +footer: "ยฉ 2026 NeuralDB, Inc. Documentation licensed under CC BY 4.0." diff --git a/sample-sites/neuraldb-docs/index.html b/sample-sites/neuraldb-docs/index.html new file mode 100644 index 0000000..1466873 --- /dev/null +++ b/sample-sites/neuraldb-docs/index.html @@ -0,0 +1,2784 @@ + + + + + + + +MD-CMS + + + + + + + + + + + + + + +
+ + + + + + diff --git a/sample-sites/neuraldb-docs/nav.yml b/sample-sites/neuraldb-docs/nav.yml new file mode 100644 index 0000000..cf82352 --- /dev/null +++ b/sample-sites/neuraldb-docs/nav.yml @@ -0,0 +1,240 @@ +# nav.yml โ€” generated by mdcms.py +sections: + - code: overview + defaultname: Overview + sort: 100 + pagesvisibility: visible + + - code: installation + defaultname: Installation + sort: 200 + pagesvisibility: visible + + - code: configuration + defaultname: Configuration + sort: 300 + pagesvisibility: visible + + - code: query-language + defaultname: Query Language + sort: 400 + pagesvisibility: visible + + - code: client-sdks + defaultname: Client SDKs + sort: 500 + pagesvisibility: visible + + - code: operations + defaultname: Operations + sort: 600 + pagesvisibility: visible + +pages: + - file: pages/index.md + title: What is NeuralDB? + section-id: overview + sort: 100 + variants: [en] + titles: + en: What is NeuralDB? + + - file: pages/concepts.md + title: Core Concepts + section-id: overview + sort: 110 + variants: [en] + titles: + en: Core Concepts + + - file: pages/architecture.md + title: Architecture + section-id: overview + sort: 120 + variants: [en] + titles: + en: Architecture + + - file: pages/comparison.md + title: Comparison + section-id: overview + sort: 130 + variants: [en] + titles: + en: Comparison + + - file: pages/install-docker.md + title: Docker Install + section-id: installation + sort: 100 + variants: [en] + titles: + en: Docker Install + + - file: pages/install-kubernetes.md + title: Kubernetes + section-id: installation + sort: 110 + variants: [en] + titles: + en: Kubernetes + + - file: pages/install-cloud.md + title: Cloud Managed + section-id: installation + sort: 120 + variants: [en] + titles: + en: Cloud Managed + + - file: pages/install-local.md + title: Local Development + section-id: installation + sort: 130 + variants: [en] + titles: + en: Local Development + + - file: pages/config-server.md + title: Server Config + section-id: configuration + sort: 100 + variants: [en] + titles: + en: Server Config + + - file: pages/config-auth.md + title: Authentication + section-id: configuration + sort: 110 + variants: [en] + titles: + en: Authentication + + - file: pages/config-storage.md + title: Storage Config + section-id: configuration + sort: 120 + variants: [en] + titles: + en: Storage Config + + - file: pages/config-replication.md + title: Replication + section-id: configuration + sort: 130 + variants: [en] + titles: + en: Replication + + - file: pages/nql-basics.md + title: NQL Basics + section-id: query-language + sort: 100 + variants: [en] + titles: + en: NQL Basics + + - file: pages/nql-vectors.md + title: Vector Queries + section-id: query-language + sort: 110 + variants: [en] + titles: + en: Vector Queries + + - file: pages/nql-hybrid.md + title: Hybrid Queries + section-id: query-language + sort: 120 + variants: [en] + titles: + en: Hybrid Queries + + - file: pages/nql-aggregations.md + title: Aggregations + section-id: query-language + sort: 130 + variants: [en] + titles: + en: Aggregations + + - file: pages/nql-transactions.md + title: Transactions + section-id: query-language + sort: 140 + variants: [en] + titles: + en: Transactions + + - file: pages/sdk-python.md + title: Python SDK + section-id: client-sdks + sort: 100 + variants: [en] + titles: + en: Python SDK + + - file: pages/sdk-javascript.md + title: JavaScript SDK + section-id: client-sdks + sort: 110 + variants: [en] + titles: + en: JavaScript SDK + + - file: pages/sdk-go.md + title: Go SDK + section-id: client-sdks + sort: 120 + variants: [en] + titles: + en: Go SDK + + - file: pages/sdk-rest.md + title: REST API + section-id: client-sdks + sort: 130 + variants: [en] + titles: + en: REST API + + - file: pages/ops-monitoring.md + title: Monitoring + section-id: operations + sort: 100 + variants: [en] + titles: + en: Monitoring + + - file: pages/ops-backup.md + title: Backup & Restore + section-id: operations + sort: 110 + variants: [en] + titles: + en: Backup & Restore + + - file: pages/ops-scaling.md + title: Scaling + section-id: operations + sort: 120 + variants: [en] + titles: + en: Scaling + + - file: pages/ops-migration.md + title: Migration + section-id: operations + sort: 130 + variants: [en] + titles: + en: Migration + + - file: pages/ops-troubleshooting.md + title: Troubleshooting + section-id: operations + sort: 140 + variants: [en] + titles: + en: Troubleshooting diff --git a/sample-sites/neuraldb-docs/pages/architecture.md b/sample-sites/neuraldb-docs/pages/architecture.md new file mode 100644 index 0000000..a2d46c5 --- /dev/null +++ b/sample-sites/neuraldb-docs/pages/architecture.md @@ -0,0 +1,188 @@ +--- +title: Architecture +sort: 120 +section-id: overview +keywords: architecture, storage engine, query planner, replication, WAL, HNSW +description: NeuralDB internal architecture โ€” storage engine, query planner, and replication +language: en +--- + +# Architecture + +![NeuralDB Architecture](assets/images/architecture.jpg) + +NeuralDB is built on a custom storage engine that co-locates relational and vector data, with a query planner that understands both SQL predicates and vector similarity operations natively. + +## High-Level Architecture + +``` +Client (psql / SDK / REST) + โ”‚ + โ–ผ +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ Connection Layer โ”‚ +โ”‚ (PostgreSQL Wire Protocol compatible) โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ + โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ–ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” + โ”‚ Query Parser โ”‚ + โ”‚ (SQL + NQL ext.) โ”‚ + โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ + โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ–ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” + โ”‚ Semantic Planner โ”‚โ—„โ”€โ”€ Statistics + Index Metadata + โ”‚ (hybrid cost model) โ”‚ + โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ + โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ–ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” + โ”‚ Execution Engine โ”‚ + โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ + โ”‚ โ”‚ SQL Executor โ”‚ โ”‚ + โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ + โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ + โ”‚ โ”‚ ANN Executor โ”‚ โ”‚ + โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ + โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ + โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ–ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” + โ”‚ Storage Engine โ”‚ + โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ + โ”‚ โ”‚ Row Store โ”‚ โ”‚โ—„โ”€โ”€ SST Files (columnar) + โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ + โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ + โ”‚ โ”‚ Vector Store โ”‚ โ”‚โ—„โ”€โ”€ HNSW Graph Files + โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ + โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ + โ”‚ โ”‚ WAL โ”‚ โ”‚โ—„โ”€โ”€ Write-Ahead Log + โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ + โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ +``` + +## Storage Engine + +### Row Store + +NeuralDB's row store uses a Log-Structured Merge-tree (LSM) architecture inspired by RocksDB. Data is written to an in-memory write buffer (MemTable), which is periodically flushed to sorted string tables (SSTables) on disk. Background compaction merges SSTables and reclaims space. + +Key properties: +- **Write-optimised**: writes are sequential, not random โ€” excellent NVMe utilisation +- **Columnar format**: SSTables store data column-by-column for fast analytical scans +- **Compression**: LZ4 by default, Zstd for archival storage โ€” typically 3โ€“6ร— compression ratio + +### Vector Store + +Vectors are stored separately from rows in a Vector Store. The Vector Store maintains: + +1. **Raw vector data** โ€” the float32 arrays, stored in compressed pages +2. **HNSW graph** โ€” the in-memory navigation graph for ANN search + +The HNSW graph is loaded into memory on startup and kept warm. Memory required โ‰ˆ `num_vectors ร— dimensions ร— 4 bytes ร— 1.3` (1.3ร— overhead for the graph structure). + +For a 10M-row table with 1536-dimensional embeddings: `10M ร— 1536 ร— 4 ร— 1.3 โ‰ˆ 80 GB`. Plan memory accordingly. + +### Write-Ahead Log (WAL) + +All writes (row and vector) are first written to the WAL before being applied to the storage engine. The WAL provides: + +- **Durability**: committed transactions survive crashes +- **Replication**: replicas apply the WAL stream from the primary +- **Point-in-time recovery (PITR)**: archive the WAL to recover to any point in time + +WAL segments are 128 MB by default and are archived to the configured storage backend (local disk, S3, GCS) upon rotation. + +## Query Planner + +The Semantic Planner extends a PostgreSQL-compatible query planner with understanding of vector operations. + +### Hybrid Cost Model + +For hybrid queries (vector + relational), the planner considers two physical plans: + +**Plan A: Pre-filter** +``` +Filter(price < 100) โ†’ ANN(embedding, k=10) +``` +Cost: selectivity ร— full_scan_cost + ANN_cost(filtered_set) + +**Plan B: Post-filter** +``` +ANN(embedding, k=10ร—estimated_filter_ratio) โ†’ Filter(price < 100) +``` +Cost: ANN_cost(full_index) + filter_cost + +The planner uses column statistics (histogram, null fraction, distinct values) and vector index parameters to estimate costs. It picks the plan with the lower estimated cost. + +### Index Types + +NeuralDB supports the following index types: + +| Index | Data | Purpose | +|-------|------|---------| +| B-tree | Scalar columns | Equality, range queries | +| Hash | Scalar columns | Equality only (faster than B-tree) | +| GIN | JSON, arrays | Containment queries | +| HNSW | VECTOR columns | Approximate nearest neighbour | +| IVF-Flat | VECTOR columns | High-recall exact-ish search | +| BRIN | Timestamp columns | Range scans on append-only data | + +## Replication + +NeuralDB uses streaming replication. The primary continuously ships WAL segments to replicas, which apply them in order. + +### Synchronous vs Asynchronous Replication + +```sql +-- Set replication mode per-transaction +SET synchronous_commit = 'on'; -- wait for WAL to reach all sync replicas (safest) +SET synchronous_commit = 'local'; -- wait for local WAL flush only (faster) +SET synchronous_commit = 'off'; -- don't wait (fastest, small durability window) +``` + +### Read Replicas + +Replicas accept `SELECT` queries. Direct read-heavy workloads to replicas: + +``` +primary: write queries + critical reads +replica-1: analytical queries, reporting +replica-2: search API traffic +``` + +The client SDK supports automatic read/write splitting: + +```python +client = NeuralDB( + primary="primary.example.com:5432", + replicas=["replica1.example.com:5432", "replica2.example.com:5432"], + read_from="replicas", + replica_selection="round-robin", +) +``` + +## Memory Architecture + +NeuralDB divides available memory into three pools: + +| Pool | Purpose | Default | +|------|---------|---------| +| `shared_buffers` | Row store page cache | 25% of RAM | +| `vector_buffer` | HNSW graph warm cache | 40% of RAM | +| `work_mem` | Per-query sort and hash buffers | 64 MB | + +Tune these in `neuraldb.conf`: + +```ini +shared_buffers = 8GB +vector_buffer = 16GB +work_mem = 128MB +``` + +## Consistency Model + +NeuralDB provides **strong consistency** for primary reads and **eventual consistency** for replica reads (with a configurable replication lag threshold). + +Reads on the primary always see the latest committed data. Reads on replicas may lag behind the primary by the `max_replication_lag` setting (default: 1 second). To force a replica to wait until it is caught up: + +```sql +SELECT pg_wait_for_replica_replay('0/1234ABCD'); +``` diff --git a/sample-sites/neuraldb-docs/pages/comparison.md b/sample-sites/neuraldb-docs/pages/comparison.md new file mode 100644 index 0000000..332e896 --- /dev/null +++ b/sample-sites/neuraldb-docs/pages/comparison.md @@ -0,0 +1,102 @@ +--- +title: Comparison +sort: 130 +section-id: overview +keywords: comparison, Postgres, pgvector, Pinecone, Weaviate, MongoDB, alternatives +description: How NeuralDB compares to Postgres+pgvector, Pinecone, Weaviate, and MongoDB Atlas Vector Search +language: en +--- + +# Comparison + +This page provides an honest comparison of NeuralDB against the most common alternatives for applications that need vector search. + +## NeuralDB vs PostgreSQL + pgvector + +pgvector is the most popular way to add vector search to an existing PostgreSQL deployment. If you already run Postgres, the low barrier to entry is attractive. Here is where the two diverge: + +| Feature | NeuralDB | Postgres + pgvector | +|---------|---------|---------------------| +| Vector index algorithm | HNSW (native) | HNSW, IVFFlat | +| Max dimensions | 65,536 | 16,000 | +| Index build speed | Native Rust (fast) | C extension (moderate) | +| Parallel index builds | Yes | Limited | +| Vector data memory isolation | Dedicated vector buffer pool | Shared with row pages | +| Horizontal sharding | Built-in | Manual (Citus, Patroni) | +| Automatic embeddings | Yes | No | +| Multi-modal vectors | Yes | No (one VECTOR column type) | +| Streaming ingestion | Yes | No | +| Read replica for vector queries | Automatic routing | Manual | + +**When to choose Postgres + pgvector:** You already have a Postgres deployment, your dataset is under 10M vectors, and you do not need automatic embeddings or horizontal scaling. The operational overhead of a new database system is not worth it. + +**When to choose NeuralDB:** Your vector dataset exceeds 10M rows, you need horizontal sharding, you want automatic embedding pipelines, or you are starting a new project and want a purpose-built system. + +## NeuralDB vs Pinecone + +Pinecone is a fully managed, purpose-built vector database. It excels at pure vector search at massive scale. + +| Feature | NeuralDB | Pinecone | +|---------|---------|---------| +| Relational data | Full SQL | Metadata filters only | +| Hybrid queries | Single query, query planner | Metadata post-filter | +| ACID transactions | Yes | No | +| SQL interface | Yes | Proprietary API | +| Self-hosted option | Yes | No | +| Pricing model | Infrastructure cost | Per-request + storage | +| Latency (p99, 1M vectors) | ~5ms | ~10ms (managed) | +| Data gravity | Stays in your infra | Vendor-managed | + +**When to choose Pinecone:** You need a fully managed solution with no operational overhead, your workload is pure vector search with simple metadata filtering, and you are comfortable with a vendor-specific API and pricing model. + +**When to choose NeuralDB:** You need relational data co-located with vectors, ACID transactions, SQL compatibility, self-hosting, or lower total cost of ownership at scale. + +## NeuralDB vs Weaviate + +Weaviate is an open-source vector database with a GraphQL-based query language and built-in module support for embedding generation. + +| Feature | NeuralDB | Weaviate | +|---------|---------|---------| +| Query language | SQL (NQL) | GraphQL | +| Relational joins | Yes | No | +| ACID transactions | Yes | Eventually consistent | +| SQL wire compatibility | PostgreSQL wire protocol | Proprietary | +| Embedding modules | Yes | Yes (vectorizers) | +| BM25 hybrid search | Yes | Yes | +| Multi-tenancy | Row-level, schema-level | Class-level | +| Replication | Sync + async | Eventual | + +**When to choose Weaviate:** You want an open-source solution with a rich ecosystem of vectorizer modules and a GraphQL interface. If your team is more comfortable with graph-shaped queries than SQL, Weaviate is a natural fit. + +**When to choose NeuralDB:** You need SQL, transactional guarantees, relational joins between your vector data and other structured data, or PostgreSQL wire protocol compatibility (so existing tools like dbt, Metabase, and psql work out of the box). + +## NeuralDB vs MongoDB Atlas Vector Search + +MongoDB Atlas added vector search as an extension to its document model. It is a convenient choice if you already run Atlas. + +| Feature | NeuralDB | MongoDB Atlas Vector Search | +|---------|---------|---------------------------| +| Data model | Relational + vector | Document + vector | +| Query language | SQL | MQL (MongoDB Query Language) | +| ACID transactions | Yes (all operations) | Yes (within a session) | +| Horizontal scaling | Native sharding | Atlas sharding | +| Vector index type | HNSW | ENN (exact), HNSW | +| Full-text + vector hybrid | Yes | Yes (Atlas Search) | +| Self-hosted | Yes | Atlas only | + +**When to choose MongoDB Atlas Vector Search:** Your application already uses MongoDB and you want to add vector search without changing your data model or infrastructure. The document model maps well to semi-structured data. + +**When to choose NeuralDB:** You need relational data integrity, SQL, lower query latency, or the ability to self-host. If your data is inherently tabular (rather than document-shaped), NeuralDB's relational model will be a better fit. + +## Performance Benchmarks + +The following benchmarks were run against 10M 1536-dimensional vectors on equivalent hardware (32 vCPU, 128 GB RAM, NVMe SSD): + +| System | QPS (recall@95%) | p50 latency | p99 latency | Index build time | +|--------|-----------------|-------------|-------------|-----------------| +| NeuralDB 1.0 | 8,400 | 1.2ms | 4.8ms | 22 min | +| pgvector 0.7 | 3,100 | 2.9ms | 12ms | 45 min | +| Pinecone (s1) | 5,200 | 1.8ms | 8ms | Managed | +| Weaviate 1.24 | 4,600 | 2.1ms | 9ms | 31 min | + +Benchmarks are inherently workload-dependent. Run your own benchmarks against your specific data and query patterns before making infrastructure decisions. diff --git a/sample-sites/neuraldb-docs/pages/concepts.md b/sample-sites/neuraldb-docs/pages/concepts.md new file mode 100644 index 0000000..0ec21b3 --- /dev/null +++ b/sample-sites/neuraldb-docs/pages/concepts.md @@ -0,0 +1,171 @@ +--- +title: Core Concepts +sort: 110 +section-id: overview +keywords: concepts, vectors, embeddings, hybrid queries, nodes, HNSW, ANN +description: Core NeuralDB concepts โ€” vectors, embeddings, hybrid queries, and the node model +language: en +--- + +# Core Concepts + +Understanding these fundamental concepts will help you use NeuralDB effectively and make good architectural decisions for your application. + +## Vectors and Embeddings + +A **vector** is an ordered list of floating-point numbers โ€” a point in high-dimensional space. In NeuralDB, vectors are used to represent the semantic meaning of data. + +An **embedding** is a vector produced by a machine learning model that encodes the semantic meaning of its input. Similar inputs produce vectors that are close together in the embedding space. For example, the sentences "I love dogs" and "I adore canines" will produce embeddings that are close to each other, even though they share no words. + +NeuralDB stores embeddings as `VECTOR(n)` columns, where `n` is the dimensionality (the number of float32 values). Common dimensionalities: + +| Model | Dimensions | +|-------|-----------| +| OpenAI text-embedding-3-small | 1536 | +| OpenAI text-embedding-3-large | 3072 | +| Cohere embed-english-v3.0 | 1024 | +| Google text-embedding-004 | 768 | +| BAAI/bge-m3 | 1024 | + +## Distance Metrics + +NeuralDB computes similarity between two vectors using one of three distance metrics: + +### Cosine Similarity + +Measures the angle between two vectors. Ranges from -1 (opposite) to 1 (identical). Ideal for text embeddings produced by models trained with cosine objectives: + +``` +cosine_similarity(a, b) = (a ยท b) / (|a| ร— |b|) +``` + +In NQL, use the `<=>` operator or `COSINE_SIMILARITY()` function. + +### Dot Product + +Measures the product of vector magnitudes and the angle between them. Used when vectors are not normalised and magnitude carries information (e.g., collaborative filtering): + +``` +dot_product(a, b) = ฮฃ(aแตข ร— bแตข) +``` + +In NQL, use the `<#>` operator or `DOT_PRODUCT()` function. + +### Euclidean Distance (L2) + +Measures straight-line distance between two points. Lower is more similar. Useful for spatial data and image embeddings: + +``` +l2_distance(a, b) = โˆš(ฮฃ(aแตข - bแตข)ยฒ) +``` + +In NQL, use the `<->` operator or `L2_DISTANCE()` function. + +## Vector Indexes + +NeuralDB builds vector indexes using the **HNSW (Hierarchical Navigable Small World)** algorithm. HNSW provides: + +- Sub-linear approximate nearest neighbour (ANN) search +- Configurable trade-off between speed and recall +- Incremental updates (no full rebuild needed when inserting) + +### HNSW Parameters + +When creating a vector index: + +```sql +CREATE INDEX ON documents +USING hnsw (embedding vector_cosine_ops) +WITH (m = 16, ef_construction = 64); +``` + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `m` | Number of bi-directional links per node. Higher = better recall, more memory | 16 | +| `ef_construction` | Size of candidate set during index construction. Higher = better quality, slower build | 64 | +| `ef_search` | Size of candidate set at query time. Set per-query with `SET hnsw.ef_search = 100` | 40 | + +### Exact vs Approximate Search + +By default, vector queries use the HNSW index (approximate). For exact results (slower but 100% recall), use: + +```sql +SET neuraldb.vector_scan = 'exact'; +SELECT * FROM documents ORDER BY embedding <=> :query LIMIT 10; +``` + +## Hybrid Queries + +A hybrid query combines vector similarity with relational predicates in a single query plan. The NeuralDB query planner evaluates two strategies and picks the cheaper one: + +1. **Pre-filter then search** โ€” apply relational filters first to reduce the candidate set, then run ANN search on the filtered set +2. **Post-filter** โ€” run ANN search to get top-k candidates, then apply relational filters + +NeuralDB automatically selects the optimal strategy based on selectivity estimates. You can hint the planner: + +```sql +SELECT * FROM documents +WHERE /*+ PREFILTER */ category = 'news' +ORDER BY embedding <=> :query +LIMIT 10; +``` + +## Tables and Schemas + +NeuralDB is schema-based, like PostgreSQL. Everything lives inside a database โ†’ schema โ†’ table hierarchy: + +```sql +CREATE DATABASE my_app; +\c my_app + +CREATE SCHEMA vectors; +CREATE SCHEMA metadata; + +CREATE TABLE vectors.documents ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + content TEXT NOT NULL, + embedding VECTOR(1536), + schema_id TEXT REFERENCES metadata.schemas(id), + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() +); +``` + +## Nodes + +NeuralDB is a distributed system. A **node** is a single NeuralDB process. Nodes have one of three roles: + +| Role | Responsibilities | +|------|----------------| +| **Primary** | Accepts reads and writes, coordinates transactions | +| **Replica** | Accepts reads, replicates writes from primary | +| **Index** | Maintains vector indexes for shard(s), offloads ANN queries | + +In a single-node deployment, one process takes all three roles. + +## Sharding + +NeuralDB shards data by `shard_key`. By default, the primary key is used as the shard key: + +```sql +CREATE TABLE events ( + id UUID PRIMARY KEY, + tenant_id UUID NOT NULL, + event_type TEXT, + embedding VECTOR(768) +) SHARD BY tenant_id; +``` + +All rows with the same `tenant_id` are guaranteed to reside on the same shard, which enables efficient tenant-scoped queries without cross-shard joins. + +## Transactions + +NeuralDB uses multi-version concurrency control (MVCC) for transaction isolation, identical to PostgreSQL: + +```sql +BEGIN; + INSERT INTO documents (content, embedding) VALUES ($1, $2); + UPDATE document_counts SET count = count + 1 WHERE id = $3; +COMMIT; +``` + +Both the vector data and the relational data are committed atomically. If the transaction rolls back, neither the row nor the vector index entry is committed. diff --git a/sample-sites/neuraldb-docs/pages/config-auth.md b/sample-sites/neuraldb-docs/pages/config-auth.md new file mode 100644 index 0000000..fa92c2d --- /dev/null +++ b/sample-sites/neuraldb-docs/pages/config-auth.md @@ -0,0 +1,223 @@ +--- +title: Authentication +sort: 110 +section-id: configuration +keywords: authentication, API keys, mTLS, RBAC, SSO, pg_hba, security +description: Configuring NeuralDB authentication โ€” API keys, mTLS, role-based access control, and SSO +language: en +--- + +# Authentication + +NeuralDB supports multiple authentication methods, from simple password auth to mutual TLS and enterprise SSO. This page explains how to configure each. + +## Host-Based Authentication (pg_hba.conf) + +The `pg_hba.conf` file controls which clients can connect and how they must authenticate. It is evaluated top-to-bottom and the first matching rule applies. + +``` +# pg_hba.conf +# FORMAT: TYPE DATABASE USER ADDRESS METHOD + +# Local Unix socket connections (no password needed for postgres superuser) +local all neuraldb trust +local all all md5 + +# IPv4 connections from local network +host all all 127.0.0.1/32 scram-sha-256 +host all all 10.0.0.0/8 scram-sha-256 + +# Reject all other connections +host all all 0.0.0.0/0 reject +``` + +Reload after changes: + +```sql +SELECT pg_reload_conf(); +``` + +## Authentication Methods + +| Method | Security | Use case | +|--------|---------|---------| +| `trust` | None | Local socket, trusted environments | +| `md5` | Weak | Legacy compatibility | +| `scram-sha-256` | Strong (default) | Standard password auth | +| `cert` | Very strong | Mutual TLS authentication | +| `ldap` | Strong | Enterprise LDAP/AD integration | +| `radius` | Strong | RADIUS server | +| `gss` | Strong | Kerberos | +| `jwt` | Strong | Token-based auth | + +### Enabling scram-sha-256 + +```ini +# neuraldb.conf +password_encryption = scram-sha-256 +``` + +Set passwords for users: + +```sql +CREATE USER app_user WITH PASSWORD 'secure-random-password'; +ALTER USER app_user PASSWORD 'new-secure-password'; +``` + +## Role-Based Access Control (RBAC) + +### Creating Roles + +```sql +-- Application read-only role +CREATE ROLE app_readonly; +GRANT CONNECT ON DATABASE myapp TO app_readonly; +GRANT USAGE ON SCHEMA public TO app_readonly; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO app_readonly; +ALTER DEFAULT PRIVILEGES IN SCHEMA public + GRANT SELECT ON TABLES TO app_readonly; + +-- Application read-write role +CREATE ROLE app_readwrite; +GRANT app_readonly TO app_readwrite; +GRANT INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO app_readwrite; +ALTER DEFAULT PRIVILEGES IN SCHEMA public + GRANT INSERT, UPDATE, DELETE ON TABLES TO app_readwrite; + +-- Vector operations role (needed for ANN searches) +CREATE ROLE vector_user; +GRANT USAGE ON SCHEMA public TO vector_user; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO vector_user; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO vector_user; + +-- Application user +CREATE USER my_app WITH PASSWORD 'app-password'; +GRANT app_readwrite TO my_app; +GRANT vector_user TO my_app; +``` + +### Row-Level Security (RLS) + +For multi-tenant applications, use Row-Level Security to enforce tenant isolation at the database level: + +```sql +ALTER TABLE documents ENABLE ROW LEVEL SECURITY; + +-- Tenants can only see their own documents +CREATE POLICY tenant_isolation ON documents + USING (tenant_id = current_setting('app.tenant_id')::UUID); + +-- Admin can see everything +CREATE POLICY admin_access ON documents + TO admin_role + USING (true); +``` + +In your application, set the tenant context before querying: + +```python +with connection.cursor() as cursor: + cursor.execute("SET app.tenant_id = %s", [tenant_id]) + cursor.execute("SELECT * FROM documents ORDER BY embedding <=> %s LIMIT 10", [embedding]) +``` + +## API Key Authentication + +NeuralDB supports an API key table for token-based authentication โ€” useful for microservices and CI pipelines that cannot use password auth: + +```sql +-- Enable the API key extension +CREATE EXTENSION neuraldb_apikeys; + +-- Create an API key for a service account +SELECT neuraldb_apikeys.create_key( + label => 'my-service', + role => 'app_readwrite' +); +-- Returns: ndb_live_abcdefghij123456... +``` + +Clients authenticate by passing the key in the connection string: + +``` +postgresql://apikey:ndb_live_abc123@host:5432/mydb +``` + +Revoke a key: + +```sql +SELECT neuraldb_apikeys.revoke_key('ndb_live_abc123'); +``` + +## Mutual TLS (mTLS) + +For the highest security, require clients to present a certificate signed by your CA. + +### 1. Generate a CA + +```bash +# Generate CA key and certificate +openssl genrsa -out ca.key 4096 +openssl req -new -x509 -key ca.key -out ca.crt -days 3650 \ + -subj "/CN=NeuralDB CA/O=My Org" +``` + +### 2. Issue a Server Certificate + +```bash +openssl genrsa -out server.key 2048 +openssl req -new -key server.key -out server.csr \ + -subj "/CN=neuraldb.example.com" +openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key \ + -CAcreateserial -out server.crt -days 365 +``` + +### 3. Configure NeuralDB + +```ini +# neuraldb.conf +ssl = on +ssl_cert_file = '/etc/neuraldb/ssl/server.crt' +ssl_key_file = '/etc/neuraldb/ssl/server.key' +ssl_ca_file = '/etc/neuraldb/ssl/ca.crt' +ssl_crl_file = '/etc/neuraldb/ssl/crl.pem' # optional certificate revocation list +``` + +### 4. Require Client Certificates + +``` +# pg_hba.conf +hostssl all all 0.0.0.0/0 cert clientcert=verify-full +``` + +## LDAP / Active Directory + +``` +# pg_hba.conf +host all all 0.0.0.0/0 ldap \ + ldapserver=ldap.example.com \ + ldapport=636 \ + ldaptls=1 \ + ldapbasedn="dc=example,dc=com" \ + ldapbinddn="cn=neuraldb,dc=example,dc=com" \ + ldapbindpasswd=bindpassword \ + ldapsearchfilter="(sAMAccountName=%s)" +``` + +## Auditing + +Enable connection and statement auditing: + +```sql +CREATE EXTENSION pgaudit; +``` + +```ini +# neuraldb.conf +pgaudit.log = 'ddl, write, role' +pgaudit.log_catalog = on +pgaudit.log_client = on +pgaudit.log_level = log +``` + +Audit logs are written to the standard NeuralDB log file in a structured format, suitable for ingestion by SIEM systems. diff --git a/sample-sites/neuraldb-docs/pages/config-replication.md b/sample-sites/neuraldb-docs/pages/config-replication.md new file mode 100644 index 0000000..dfa2f09 --- /dev/null +++ b/sample-sites/neuraldb-docs/pages/config-replication.md @@ -0,0 +1,185 @@ +--- +title: Replication +sort: 130 +section-id: configuration +keywords: replication, primary, replica, streaming replication, multi-region, consistency +description: Configuring NeuralDB replication โ€” primary/replica setup, multi-region, and consistency levels +language: en +--- + +# Replication + +NeuralDB replication is based on streaming replication: the primary continuously ships WAL records to replicas, which apply them in real time. This page explains how to set up and configure replication. + +## Prerequisites + +- The primary must have `wal_level = replica` or higher +- `max_wal_senders` must be greater than the number of replicas +- A replication user must exist + +## Setting Up a Primary + +Configure `neuraldb.conf` on the primary: + +```ini +# neuraldb.conf (primary) +wal_level = replica +max_wal_senders = 10 +max_replication_slots = 10 +wal_keep_size = 1GB +hot_standby_feedback = on # prevents primary from vacuuming rows still needed by replicas +``` + +Create a replication user: + +```sql +CREATE USER replicator WITH REPLICATION PASSWORD 'repl-password'; +``` + +Allow the replica to connect: + +``` +# pg_hba.conf (primary) +host replication replicator replica-ip/32 scram-sha-256 +``` + +## Setting Up a Replica + +On the replica server, use `pg_basebackup` to clone the primary: + +```bash +# On the replica server +pg_basebackup \ + --host=primary.example.com \ + --port=5432 \ + --username=replicator \ + --pgdata=/var/lib/neuraldb/data \ + --wal-method=stream \ + --checkpoint=fast \ + --progress \ + --write-recovery-conf +``` + +The `--write-recovery-conf` flag creates a `standby.signal` file and writes connection info to `postgresql.auto.conf`, which tells NeuralDB to start in standby mode. + +Configure `neuraldb.conf` on the replica: + +```ini +# neuraldb.conf (replica) +hot_standby = on # allow read queries +hot_standby_feedback = on # send feedback to primary +wal_receiver_timeout = 60s +recovery_min_apply_delay = 0 # apply WAL immediately (increase for delayed replicas) +``` + +Start the replica: + +```bash +systemctl start neuraldb +``` + +Verify replication is working: + +```sql +-- On the primary +SELECT client_addr, state, sent_lsn, write_lsn, flush_lsn, replay_lsn, + (sent_lsn - replay_lsn) AS replication_lag_bytes +FROM pg_stat_replication; +``` + +## Replication Slots + +Replication slots ensure the primary retains WAL until the replica has consumed it. This prevents the replica from falling too far behind, but also prevents WAL from being cleaned up if the replica disconnects. + +```sql +-- Create a replication slot +SELECT pg_create_physical_replication_slot('replica_1'); + +-- List slots and their lag +SELECT slot_name, active, pg_size_pretty(pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn)) AS lag +FROM pg_replication_slots; + +-- Drop a slot (do this if a replica is permanently removed) +SELECT pg_drop_replication_slot('replica_1'); +``` + +**Warning:** Monitor slot lag. An inactive slot with large lag will cause unbounded WAL accumulation and can fill your disk. + +## Synchronous Replication + +By default, replication is asynchronous โ€” the primary does not wait for replicas to acknowledge writes. For zero data loss, configure synchronous replication: + +```ini +# neuraldb.conf (primary) +synchronous_standby_names = 'FIRST 1 (replica1, replica2)' +# ^ Wait for at least 1 of the listed standbys to acknowledge each commit +``` + +Modes: +- `FIRST n (list)` โ€” wait for the first n standbys in the list +- `ANY n (list)` โ€” wait for any n standbys from the list +- `*` โ€” wait for all standbys + +Per-transaction override: + +```sql +SET synchronous_commit = 'local'; -- this transaction doesn't wait for replicas +``` + +## Multi-Region Replication + +For global deployments, replicate to remote regions. The configuration is identical to local replication, but network latency affects synchronous commit performance. + +Recommended approach for multi-region: + +``` +Primary (us-east-1) +โ”œโ”€โ”€ Sync replica (us-east-1-az2) โ† HA within region, ~2ms latency +โ”œโ”€โ”€ Async replica (eu-west-1) โ† EU reads, ~80ms latency +โ””โ”€โ”€ Async replica (ap-northeast-1) โ† APAC reads, ~170ms latency +``` + +```ini +# Synchronous only within region; async to remote regions +synchronous_standby_names = 'FIRST 1 (local_replica)' +``` + +Configure the remote replicas with a `primary_conninfo` pointing to the primary: + +```ini +# standby.signal (on replica) +primary_conninfo = 'host=primary.us-east-1.example.com port=5432 user=replicator password=repl-password sslmode=require' +``` + +## Failover + +NeuralDB does not include automatic failover out of the box. Use one of: + +- **Patroni** โ€” industry-standard HA manager for PostgreSQL-compatible databases +- **NeuralDB HA Operator** โ€” Kubernetes operator with automatic failover (see [Kubernetes docs](install-kubernetes.md)) +- **repmgr** โ€” lightweight failover manager + +Manual failover: + +```bash +# On the replica that will become the new primary +neuraldb-cli -c "SELECT pg_promote();" +``` + +After promotion, update `primary_conninfo` on all other replicas to point to the new primary. + +## Monitoring Replication + +```sql +-- Replication lag in bytes and seconds +SELECT client_addr, state, + pg_size_pretty(sent_lsn - replay_lsn) AS lag_bytes, + now() - pg_last_xact_replay_timestamp() AS lag_time +FROM pg_stat_replication; + +-- On a replica: check its own lag +SELECT now() - pg_last_xact_replay_timestamp() AS lag, + pg_is_in_recovery() AS is_replica; +``` + +Set up an alert when replication lag exceeds 30 seconds. diff --git a/sample-sites/neuraldb-docs/pages/config-server.md b/sample-sites/neuraldb-docs/pages/config-server.md new file mode 100644 index 0000000..9e57583 --- /dev/null +++ b/sample-sites/neuraldb-docs/pages/config-server.md @@ -0,0 +1,228 @@ +--- +title: Server Config +sort: 100 +section-id: configuration +keywords: neuraldb.conf, server configuration, settings, parameters, tuning +description: Complete reference for neuraldb.conf โ€” all server configuration settings explained +language: en +--- + +# Server Config + +NeuralDB is configured through `neuraldb.conf`, a key-value text file. This page documents all configuration parameters. + +## Locating the Config File + +```bash +# Show the active config file path +neuraldb-cli -c "SHOW config_file;" +``` + +Default locations: +- Linux: `/etc/neuraldb/neuraldb.conf` +- macOS Homebrew: `$(brew --prefix)/etc/neuraldb/neuraldb.conf` +- Docker: `/var/lib/neuraldb/data/neuraldb.conf` + +Changes to `neuraldb.conf` require a reload (for most parameters) or a restart: + +```bash +# Reload without restart (applies most parameters) +neuraldb-cli -c "SELECT pg_reload_conf();" + +# Full restart (required for listen_addresses, port, shared_buffers, etc.) +systemctl restart neuraldb +``` + +## Connection Settings + +```ini +# Network interface to listen on +# '*' = all interfaces, 'localhost' = local only +listen_addresses = '*' + +# TCP port +port = 5432 + +# Maximum simultaneous connections +# Each connection uses ~5 MB of memory +max_connections = 100 + +# Unix domain socket directory (Linux/macOS only) +unix_socket_directories = '/var/run/neuraldb' + +# Superuser reserved connections +# Reserves this many connections exclusively for superusers +superuser_reserved_connections = 3 +``` + +## Memory Settings + +These are the most impactful parameters for performance. + +```ini +# Page cache for relational data (row store) +# Recommended: 25% of available RAM +shared_buffers = 4GB + +# Memory for HNSW vector indexes +# Recommended: 40-60% of available RAM for vector-heavy workloads +# Must be large enough to fit all active HNSW graphs +vector_buffer = 8GB + +# Per-query working memory (sorts, hash joins) +# Recommended: 64MBโ€“256MB for OLTP, more for analytical queries +# Be conservative: (max_connections ร— work_mem) should fit in RAM +work_mem = 128MB + +# Memory for DDL maintenance (CREATE INDEX, VACUUM, etc.) +maintenance_work_mem = 2GB + +# Shared memory for parallel query workers +parallel_query_mem = 512MB +``` + +## Vector Settings + +```ini +# Default HNSW ef_search parameter (candidates evaluated per query) +# Higher = better recall, slower queries +hnsw.ef_search = 40 + +# Maximum number of vectors per shard before auto-splitting +vector_shard_size = 10000000 + +# Enable approximate nearest neighbour by default (true = HNSW index) +# Set to 'exact' to always use exact search (ignores vector indexes) +vector_scan = 'approximate' + +# Number of parallel threads for HNSW index builds +hnsw.build_threads = 4 + +# Compression algorithm for stored vector data +# 'none', 'lz4', 'scalar_quantize' (lossy but 4ร— smaller) +vector_compression = 'lz4' +``` + +## WAL Settings + +```ini +# WAL logging level +# 'minimal': minimum for crash recovery +# 'replica': enables streaming replication (default) +# 'logical': enables logical replication and CDC +wal_level = replica + +# WAL segment size (change requires initdb) +wal_segment_size = 128MB + +# Maximum number of concurrent WAL sender processes +max_wal_senders = 10 + +# Number of WAL segments to keep for standby catching up +wal_keep_size = 1GB + +# Synchronise WAL to disk before acknowledging commit +# 'on': safest; 'local': only local disk; 'off': async (fastest, tiny durability risk) +synchronous_commit = on + +# Interval between WAL checkpoints +checkpoint_timeout = 5min +checkpoint_completion_target = 0.9 +max_wal_size = 4GB +``` + +## Replication Settings + +```ini +# Comma-separated list of standby names that must acknowledge writes +# Leave empty for asynchronous replication +synchronous_standby_names = '' + +# Maximum lag allowed before primary throttles writes +max_standby_lag = 30s + +# Hot standby: allow reads on replicas +hot_standby = on +``` + +## Query Planner + +```ini +# Estimated cost of a random page fetch (tune based on SSD vs HDD) +# Lower values favour index scans; higher values favour sequential scans +random_page_cost = 1.1 # NVMe SSD (default is 4.0 for HDD) + +# Effective size of the disk cache (affects planner estimates) +effective_cache_size = 24GB # ~75% of RAM + +# Enable parallel query +max_parallel_workers_per_gather = 4 +max_parallel_workers = 8 +max_worker_processes = 16 + +# Hybrid query planner behaviour +# 'auto': planner decides pre-filter vs post-filter +# 'pre-filter': always pre-filter +# 'post-filter': always post-filter +vector_hybrid_strategy = 'auto' +``` + +## Logging + +```ini +# Log destination +log_destination = 'stderr' # 'stderr', 'csvlog', 'jsonlog', 'syslog' + +# Minimum severity to log +# 'DEBUG5' (verbose) โ†’ 'INFO' โ†’ 'WARNING' โ†’ 'ERROR' โ†’ 'FATAL' +log_min_messages = WARNING + +# Log all SQL statements with duration above this threshold (ms) +# -1 = disable; 0 = log everything; 250 = log slow queries only +log_min_duration_statement = 250 + +# Log query parameters +log_parameters = off + +# Log connection events +log_connections = on +log_disconnections = on + +# Log lock waits longer than this (ms) +deadlock_timeout = 1s +log_lock_waits = on +``` + +## Full Configuration Example + +```ini +# neuraldb.conf โ€” Production settings for a 32 vCPU / 128 GB server + +listen_addresses = '*' +port = 5432 +max_connections = 500 +superuser_reserved_connections = 5 + +shared_buffers = 32GB +vector_buffer = 64GB +work_mem = 128MB +maintenance_work_mem = 4GB + +wal_level = replica +max_wal_senders = 10 +wal_keep_size = 2GB +synchronous_commit = on +checkpoint_timeout = 10min +max_wal_size = 8GB + +random_page_cost = 1.1 +effective_cache_size = 96GB +max_parallel_workers_per_gather = 8 +max_parallel_workers = 16 + +hnsw.ef_search = 80 +vector_compression = lz4 + +log_min_duration_statement = 500 +log_connections = on +``` diff --git a/sample-sites/neuraldb-docs/pages/config-storage.md b/sample-sites/neuraldb-docs/pages/config-storage.md new file mode 100644 index 0000000..dfbc82f --- /dev/null +++ b/sample-sites/neuraldb-docs/pages/config-storage.md @@ -0,0 +1,221 @@ +--- +title: Storage Config +sort: 120 +section-id: configuration +keywords: storage, memory, disk, SSD tiers, compression, tablespace, IOPS +description: Configuring NeuralDB storage โ€” memory tiers, disk layout, compression, and tablespaces +language: en +--- + +# Storage Config + +NeuralDB's performance depends heavily on storage configuration. This page covers how to optimise disk layout, configure memory tiers, enable compression, and use tablespaces for data tiering. + +## Disk Layout Recommendations + +Separate the data directory, WAL, and vector files onto different physical disks (or at least different volumes with guaranteed IOPS): + +| Directory | Recommended storage | IOPS requirement | +|-----------|--------------------|--------------------| +| `$DATADIR/base/` | NVMe SSD | High random read/write | +| `$DATADIR/wal/` | NVMe SSD (separate) | High sequential write | +| `$DATADIR/vectors/` | NVMe SSD or RAM disk | High random read | +| `$DATADIR/archive/` | HDD or object storage | Low (sequential write) | + +Configure separate paths in `neuraldb.conf`: + +```ini +# Separate WAL onto a dedicated volume +wal_directory = '/mnt/nvme-wal/neuraldb/wal' + +# Separate vector storage +vector_data_directory = '/mnt/nvme-fast/neuraldb/vectors' + +# Archive destination for WAL shipping +archive_status_directory = '/var/lib/neuraldb/archive_status' +``` + +## Memory Tiers + +NeuralDB has three distinct memory pools: + +### shared_buffers (Row Store Cache) + +The page cache for relational data. Sized at 25% of available RAM for a dedicated database server: + +```ini +shared_buffers = 32GB # for a 128 GB server +``` + +### vector_buffer (Vector Index Cache) + +Holds the HNSW graph in memory. The entire active HNSW graph must fit in `vector_buffer` for optimal performance. When the graph doesn't fit, NeuralDB falls back to disk-based graph traversal, which is 10โ€“50ร— slower. + +Calculate the required size: + +``` +vector_buffer = num_vectors ร— dimensions ร— 4 bytes ร— hnsw_overhead_factor +``` + +Where `hnsw_overhead_factor` โ‰ˆ 1.3 for default HNSW parameters (m=16). + +```sql +-- Check current vector index memory usage +SELECT table_name, index_name, + pg_size_pretty(hnsw_graph_size_bytes) AS graph_memory, + pg_size_pretty(vector_data_size_bytes) AS data_size +FROM neuraldb_stat_vector_indexes; +``` + +### work_mem (Per-Query Buffer) + +Used for in-memory sorts, hash joins, and bitmap operations. Set conservatively โ€” each query can allocate multiple `work_mem` buffers: + +```ini +# For 200 connections with typical 2 buffers per query: +# Max memory consumption: 200 ร— 2 ร— 128MB = 51 GB +work_mem = 128MB +``` + +Override per-session for analytical queries: + +```sql +SET work_mem = '2GB'; +SELECT ... complex analytical query ... +``` + +## Compression + +### Row Store Compression + +NeuralDB compresses SSTables on disk. Choose the algorithm based on your priorities: + +```ini +# Compression algorithm for row data +# 'none': no compression (fastest reads/writes, most disk) +# 'lz4': fast, moderate compression ratio (~2-3ร—) โ€” default +# 'zstd': slower compression, better ratio (~3-5ร—) +# 'zstd-9': high compression for archival (slow, ~6-8ร—) +storage_compression = lz4 + +# Compression level (for zstd only), 1-19 +storage_compression_level = 3 +``` + +### Vector Compression + +```ini +# Vector data compression +# 'none': full precision, largest storage +# 'lz4': fast, minimal precision loss +# 'scalar_quantize': reduce to 8-bit (4ร— smaller, ~1% recall loss) +# 'product_quantize': very high compression, higher recall loss +vector_compression = lz4 +``` + +To enable scalar quantisation for a specific index: + +```sql +CREATE INDEX ON documents +USING hnsw (embedding vector_cosine_ops) +WITH (quantization = 'scalar'); +``` + +Scalar-quantised indexes use 4ร— less memory and may be faster due to better cache utilisation, at a typical recall cost of 0.5โ€“2%. + +## Tablespaces + +Use tablespaces to store different tables or indexes on different volumes: + +```sql +-- Create tablespaces pointing to different mount points +CREATE TABLESPACE fast_ssd LOCATION '/mnt/nvme-fast'; +CREATE TABLESPACE bulk_hdd LOCATION '/mnt/hdd-storage'; + +-- Create a table on fast SSD +CREATE TABLE hot_documents ( + id UUID PRIMARY KEY, + content TEXT, + embedding VECTOR(1536) +) TABLESPACE fast_ssd; + +-- Move an index to a specific tablespace +CREATE INDEX ON hot_documents USING hnsw (embedding vector_cosine_ops) +TABLESPACE fast_ssd; + +-- Move old partitions to cheaper storage +ALTER TABLE documents_2024_q1 SET TABLESPACE bulk_hdd; +``` + +## Table Partitioning + +Partition large tables by time or tenant to improve query performance and manageability: + +```sql +-- Partition documents by month +CREATE TABLE documents ( + id UUID NOT NULL DEFAULT gen_random_uuid(), + content TEXT, + embedding VECTOR(1536), + tenant_id UUID, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() +) PARTITION BY RANGE (created_at); + +CREATE TABLE documents_2026_01 + PARTITION OF documents + FOR VALUES FROM ('2026-01-01') TO ('2026-02-01') + TABLESPACE fast_ssd; + +CREATE TABLE documents_2025_archive + PARTITION OF documents + FOR VALUES FROM ('2025-01-01') TO ('2026-01-01') + TABLESPACE bulk_hdd; +``` + +## VACUUM and Autovacuum + +NeuralDB inherits PostgreSQL's MVCC-based VACUUM system: + +```ini +# Autovacuum settings +autovacuum = on +autovacuum_naptime = 1min +autovacuum_vacuum_threshold = 50 +autovacuum_vacuum_scale_factor = 0.05 # vacuum when 5% of rows are dead +autovacuum_analyze_threshold = 50 +autovacuum_analyze_scale_factor = 0.02 # analyse when 2% of rows change + +# For high-write tables, increase autovacuum aggressiveness +autovacuum_vacuum_cost_delay = 2ms # default: 20ms +autovacuum_vacuum_cost_limit = 400 # default: 200 +``` + +Manually vacuum a table: + +```sql +VACUUM ANALYZE documents; -- reclaim space + update statistics +VACUUM FULL documents; -- full rewrite (blocking, very thorough) +``` + +## Monitoring Disk Usage + +```sql +-- Table sizes +SELECT table_name, + pg_size_pretty(pg_total_relation_size(quote_ident(table_name))) AS total, + pg_size_pretty(pg_relation_size(quote_ident(table_name))) AS table, + pg_size_pretty(pg_total_relation_size(quote_ident(table_name)) + - pg_relation_size(quote_ident(table_name))) AS indexes +FROM information_schema.tables +WHERE table_schema = 'public' +ORDER BY pg_total_relation_size(quote_ident(table_name)) DESC; + +-- Vector index sizes +SELECT * FROM neuraldb_stat_vector_indexes +ORDER BY hnsw_graph_size_bytes DESC; + +-- Bloat estimate +SELECT tablename, n_dead_tup, last_vacuum, last_autovacuum +FROM pg_stat_user_tables +ORDER BY n_dead_tup DESC; +``` diff --git a/sample-sites/neuraldb-docs/pages/index.md b/sample-sites/neuraldb-docs/pages/index.md new file mode 100644 index 0000000..b25f43b --- /dev/null +++ b/sample-sites/neuraldb-docs/pages/index.md @@ -0,0 +1,125 @@ +--- +title: What is NeuralDB? +sort: 100 +section-id: overview +keywords: NeuralDB, introduction, AI database, vector database, overview +description: What NeuralDB is, its key use cases, and a high-level architecture overview +language: en +--- + +# What is NeuralDB? + +![NeuralDB Platform](assets/images/hero.jpg) + +NeuralDB is an AI-native database that unifies vector storage, semantic search, and relational data management in a single, horizontally scalable system. It is designed for applications that need to combine structured data queries with the semantic understanding that modern AI models provide. + +Traditional databases store and retrieve data based on exact matches and range queries. NeuralDB adds a third dimension: **semantic proximity**. You can ask "find the 20 products most similar to this description" at the same time as "where inventory > 0 and price < 100" โ€” in a single, atomic query with full ACID guarantees. + +## Why NeuralDB Exists + +The AI application stack of 2024โ€“2026 exposed a fundamental tension in data architecture. Teams building retrieval-augmented generation (RAG) systems, recommendation engines, and semantic search needed: + +- A **vector database** (Pinecone, Weaviate, Qdrant) for embedding storage and similarity search +- A **relational database** (PostgreSQL, MySQL) for structured metadata and transactions +- A **synchronisation layer** to keep the two in sync โ€” a constant source of bugs and operational overhead + +NeuralDB replaces all three with a single system. Vectors and relational data share the same storage engine, the same transaction log, and the same query planner. There is no synchronisation problem because there is no synchronisation needed. + +## Key Capabilities + +### Hybrid Vector + Relational Queries + +```sql +SELECT id, name, price, SIMILARITY(embedding, :query_embedding) AS score +FROM products +WHERE category = 'electronics' + AND price BETWEEN 50 AND 500 + AND stock > 0 +ORDER BY score DESC +LIMIT 10; +``` + +This query uses a vector index for semantic ranking and a B-tree index for the relational filters, with the query planner choosing the optimal execution order. + +### Multiple Vector Index Types + +NeuralDB supports three similarity metrics out of the box: + +| Metric | Use case | +|--------|----------| +| Cosine similarity | Text embeddings, normalised vectors | +| Dot product | Recommendation systems (unnormalised) | +| Euclidean (L2) | Image embeddings, spatial data | + +### Full ACID Transactions + +Unlike most vector databases, NeuralDB provides full ACID guarantees โ€” including transactional upserts of both the relational row and the vector embedding simultaneously. + +### Automatic Embedding Updates + +Configure NeuralDB to call an embedding API whenever a text column changes: + +```sql +ALTER TABLE documents +ADD COLUMN embedding VECTOR(1536) +GENERATED ALWAYS AS EMBEDDING OF content +USING openai_ada_002; +``` + +NeuralDB handles the embedding pipeline automatically โ€” no application-level embedding code required. + +### Multi-Modal Vectors + +Store and query vectors from any modality โ€” text, image, audio, or custom โ€” in the same table: + +```sql +CREATE TABLE media_items ( + id UUID PRIMARY KEY, + title TEXT, + text_embedding VECTOR(1536), + image_embedding VECTOR(512), + created_at TIMESTAMP DEFAULT NOW() +); +``` + +## Use Cases + +**Retrieval-Augmented Generation (RAG)** โ€” Store your document corpus with embeddings. Query the most relevant chunks in a single round-trip and inject them into your LLM prompt. + +**Semantic Product Search** โ€” Replace keyword search with semantic search. Find products matching "comfortable running shoes for flat feet" even if those exact words don't appear in any product description. + +**Recommendation Engines** โ€” Store user preference vectors alongside item vectors. Compute collaborative-filtering recommendations with a single NQL query. + +**Anomaly Detection** โ€” Flag records whose vectors are distant from the cluster of "normal" data. + +**Duplicate Detection** โ€” Find near-duplicate records across millions of rows using approximate nearest-neighbour (ANN) search. + +**Knowledge Graphs** โ€” Store entity embeddings alongside relationship metadata for graph-enhanced retrieval. + +## NeuralDB vs Traditional Approaches + +| Capability | NeuralDB | Postgres + pgvector | Pinecone + Postgres | +|-----------|---------|---------------------|---------------------| +| Vector search | Native, HNSW | Extension, limited | Native | +| Relational queries | Full SQL | Full SQL | None (separate DB) | +| Hybrid queries | Single query | Single query | Application-layer join | +| ACID transactions | Yes | Yes | Partial | +| Horizontal sharding | Built-in | Manual (Citus) | Managed | +| Automatic embeddings | Yes | No | No | +| Streaming ingestion | Yes | No | Partial | + +## Getting Started + +The fastest way to try NeuralDB is with Docker: + +```bash +docker run -p 5432:5432 -e NEURALDB_PASSWORD=mypassword neuraldb/neuraldb:latest +``` + +Connect with any PostgreSQL-compatible client: + +```bash +psql -h localhost -U neuraldb -d neuraldb +``` + +Then read the [Core Concepts](concepts.md) to understand the NeuralDB data model, or jump to [NQL Basics](nql-basics.md) to start writing queries. diff --git a/sample-sites/neuraldb-docs/pages/install-cloud.md b/sample-sites/neuraldb-docs/pages/install-cloud.md new file mode 100644 index 0000000..160155e --- /dev/null +++ b/sample-sites/neuraldb-docs/pages/install-cloud.md @@ -0,0 +1,186 @@ +--- +title: Cloud Managed +sort: 120 +section-id: installation +keywords: cloud, managed, NeuralDB Cloud, regions, tiers, SaaS +description: Setting up NeuralDB Cloud โ€” the fully managed service with global regions and flexible tiers +language: en +--- + +# Cloud Managed + +NeuralDB Cloud is the fully managed version of NeuralDB. It handles provisioning, patching, backups, monitoring, and scaling โ€” so you can focus on building your application rather than managing database infrastructure. + +## Getting Started + +### 1. Create an Account + +Sign up at [cloud.neuraldb.io](https://cloud.neuraldb.io). You can authenticate with Google, GitHub, or an email address. + +### 2. Create a Cluster + +Click **New Cluster** and configure: + +- **Region**: choose the cloud region closest to your application servers +- **Tier**: select based on your workload requirements (see tier comparison below) +- **Storage**: initial storage allocation (can be scaled later) +- **High Availability**: enable for production workloads + +### 3. Connect + +Once the cluster is provisioned (typically under 3 minutes), your connection string appears in the dashboard: + +``` +postgresql://neuraldb:[password]@[cluster-id].cloud.neuraldb.io:5432/[database]?sslmode=require +``` + +Use this with any PostgreSQL-compatible driver or psql: + +```bash +psql "postgresql://neuraldb:mypassword@abc123.cloud.neuraldb.io:5432/mydb?sslmode=require" +``` + +## Available Regions + +NeuralDB Cloud is available in the following regions: + +| Region | Cloud Provider | Availability | +|--------|---------------|-------------| +| us-east-1 (N. Virginia) | AWS | GA | +| us-west-2 (Oregon) | AWS | GA | +| eu-west-1 (Ireland) | AWS | GA | +| eu-central-1 (Frankfurt) | AWS | GA | +| ap-northeast-1 (Tokyo) | AWS | GA | +| ap-southeast-1 (Singapore) | AWS | GA | +| us-central1 (Iowa) | GCP | Beta | +| europe-west4 (Netherlands) | GCP | Beta | +| eastus (Virginia) | Azure | Beta | + +Multi-region replication (primary in one region, read replicas in others) is available on Business and Enterprise tiers. + +## Pricing Tiers + +### Starter + +Free tier for development and experimentation. + +| Resource | Limit | +|---------|-------| +| Storage | 5 GB | +| Vector dimensions | Up to 1536 | +| Max connections | 10 | +| PITR | No | +| HA | No | +| SLA | No | + +The Starter tier automatically suspends after 7 days of inactivity and resumes on the next connection (cold start: ~30 seconds). + +### Developer + +$29/month โ€” for side projects and pre-production environments. + +| Resource | Limit | +|---------|-------| +| vCPU | 2 dedicated | +| RAM | 8 GB | +| Storage | 100 GB NVMe SSD | +| Connections | 100 | +| PITR | 7 days | +| HA | No | + +### Business + +$199/month โ€” for production workloads. + +| Resource | Limit | +|---------|-------| +| vCPU | 8 dedicated | +| RAM | 32 GB | +| Storage | 500 GB NVMe SSD (expandable) | +| Connections | 500 | +| PITR | 30 days | +| HA | Yes (1 standby) | +| Read replicas | Up to 3 | +| SLA | 99.95% | + +### Business Plus + +$599/month โ€” for large-scale production workloads. + +| Resource | Limit | +|---------|-------| +| vCPU | 32 dedicated | +| RAM | 128 GB | +| Storage | 2 TB NVMe SSD (expandable) | +| Connections | 2,000 | +| PITR | 30 days | +| HA | Yes (2 standbys) | +| Read replicas | Up to 10 | +| Multi-region replicas | Yes | +| SLA | 99.99% | + +### Enterprise + +Custom pricing โ€” for mission-critical applications with specific compliance requirements. + +- Dedicated infrastructure (no multi-tenancy) +- Custom SLAs +- Private endpoints (AWS PrivateLink, GCP Private Service Connect) +- SOC 2 Type II, HIPAA, and ISO 27001 compliance +- Dedicated support with SLA + +## Connecting from Your Application + +### Connection Pooling + +NeuralDB Cloud includes PgBouncer-based connection pooling. Use the pooler endpoint for serverless and short-lived connections: + +``` +postgresql://neuraldb:[password]@[cluster-id]-pooler.cloud.neuraldb.io:5432/[database] +``` + +| Connection type | Endpoint suffix | Use for | +|----------------|----------------|---------| +| Direct | (none) | Long-lived connections, COPY operations | +| Transaction pooling | `-pooler` | Serverless, short-lived connections | +| Session pooling | `-session-pooler` | Prepared statements | + +### IP Allow-listing + +Restrict access to specific IP ranges in the **Security** tab of your cluster dashboard. Enter your application servers' CIDR ranges (e.g., `10.0.0.0/8` for private VPC, or specific public IPs). + +### SSL/TLS + +All connections require TLS. Download the cluster CA certificate from the dashboard and verify it in your connection string: + +``` +sslmode=verify-full&sslrootcert=/path/to/ca.pem +``` + +## Monitoring and Alerting + +NeuralDB Cloud includes a built-in monitoring dashboard with: + +- Query throughput and latency percentiles (p50, p95, p99) +- Connection count +- Storage usage +- Vector index size +- Replication lag + +Configure alerts for: +- Storage > 80% full +- Average query latency > 500ms +- Replication lag > 30s +- Failed connections + +Metrics are also available via the NeuralDB Cloud API for ingestion into your own monitoring stack (Datadog, Grafana Cloud, New Relic). + +## Branching (Database Branches) + +NeuralDB Cloud supports **branching** โ€” create an instant copy-on-write clone of your production database for development, testing, or migrations: + +```bash +neuraldb-cloud branch create staging --from production +``` + +Branches share storage pages with the parent until they diverge (copy-on-write). A branch of a 100 GB database costs storage only for the pages that change. diff --git a/sample-sites/neuraldb-docs/pages/install-docker.md b/sample-sites/neuraldb-docs/pages/install-docker.md new file mode 100644 index 0000000..c6dd2e3 --- /dev/null +++ b/sample-sites/neuraldb-docs/pages/install-docker.md @@ -0,0 +1,204 @@ +--- +title: Docker Install +sort: 100 +section-id: installation +keywords: Docker, install, docker run, docker-compose, volumes, container +description: Installing NeuralDB using Docker โ€” single container and docker-compose setups +language: en +--- + +# Docker Install + +Docker is the fastest way to run NeuralDB locally or in a single-server deployment. NeuralDB's official Docker image is published to Docker Hub as `neuraldb/neuraldb`. + +## Quick Start + +Run a single NeuralDB instance: + +```bash +docker run -d \ + --name neuraldb \ + -p 5432:5432 \ + -e NEURALDB_PASSWORD=mypassword \ + -e NEURALDB_DB=mydb \ + -v neuraldb_data:/var/lib/neuraldb/data \ + neuraldb/neuraldb:latest +``` + +Connect with psql: + +```bash +psql -h localhost -p 5432 -U neuraldb -d mydb +# Password: mypassword +``` + +## Environment Variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `NEURALDB_PASSWORD` | required | Password for the `neuraldb` superuser | +| `NEURALDB_USER` | `neuraldb` | Superuser username | +| `NEURALDB_DB` | `neuraldb` | Default database name | +| `NEURALDB_PORT` | `5432` | TCP port | +| `NEURALDB_MAX_CONNECTIONS` | `100` | Maximum concurrent connections | +| `NEURALDB_SHARED_BUFFERS` | `256MB` | Row store page cache | +| `NEURALDB_VECTOR_BUFFER` | `512MB` | Vector index memory | +| `NEURALDB_WAL_LEVEL` | `replica` | WAL level (`minimal`, `replica`, `logical`) | + +## Available Tags + +| Tag | Description | +|-----|-------------| +| `latest` | Latest stable release | +| `1.0` | Specific major version | +| `1.0.3` | Specific patch version | +| `nightly` | Nightly build from main branch | +| `1.0-alpine` | Alpine-based image (smaller, less glibc compat) | + +## Volumes + +NeuralDB stores data in `/var/lib/neuraldb/data` inside the container. Always mount a named volume or bind mount to persist data: + +```bash +# Named volume (recommended) +docker volume create neuraldb_data +docker run -v neuraldb_data:/var/lib/neuraldb/data neuraldb/neuraldb:latest + +# Bind mount +docker run -v /srv/neuraldb:/var/lib/neuraldb/data neuraldb/neuraldb:latest +``` + +The data directory includes: +- `base/` โ€” table and index data +- `vectors/` โ€” HNSW graph files and raw vector data +- `wal/` โ€” write-ahead log segments +- `neuraldb.conf` โ€” runtime configuration (editable) + +## docker-compose Setup + +A production-grade docker-compose file for NeuralDB with automatic backup: + +```yaml +# docker-compose.yml +version: '3.9' + +services: + neuraldb: + image: neuraldb/neuraldb:1.0 + container_name: neuraldb + restart: unless-stopped + ports: + - "127.0.0.1:5432:5432" # bind to localhost only โ€” use nginx for external access + environment: + NEURALDB_PASSWORD: ${NEURALDB_PASSWORD} + NEURALDB_USER: ${NEURALDB_USER:-neuraldb} + NEURALDB_DB: ${NEURALDB_DB:-neuraldb} + NEURALDB_SHARED_BUFFERS: "4GB" + NEURALDB_VECTOR_BUFFER: "8GB" + NEURALDB_MAX_CONNECTIONS: "200" + volumes: + - neuraldb_data:/var/lib/neuraldb/data + - neuraldb_wal:/var/lib/neuraldb/wal + - ./neuraldb.conf:/etc/neuraldb/neuraldb.conf:ro # optional custom config + shm_size: '2gb' # increase shared memory for large sort operations + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${NEURALDB_USER:-neuraldb}"] + interval: 10s + timeout: 5s + retries: 5 + deploy: + resources: + limits: + memory: 16G + reservations: + memory: 8G + + neuraldb-backup: + image: neuraldb/neuraldb-backup:latest + environment: + NEURALDB_HOST: neuraldb + NEURALDB_PASSWORD: ${NEURALDB_PASSWORD} + S3_BUCKET: ${BACKUP_S3_BUCKET} + S3_PREFIX: backups/neuraldb/ + SCHEDULE: "0 2 * * *" # 2am daily + depends_on: + neuraldb: + condition: service_healthy + +volumes: + neuraldb_data: + driver: local + driver_opts: + type: none + o: bind + device: /srv/neuraldb/data + neuraldb_wal: + driver: local + driver_opts: + type: none + o: bind + device: /srv/neuraldb/wal +``` + +Start with: + +```bash +echo "NEURALDB_PASSWORD=$(openssl rand -base64 32)" > .env +docker-compose up -d +``` + +## Initialisation Scripts + +Place `.sql` or `.sh` scripts in `/docker-entrypoint-initdb.d/` to run them on first startup: + +```bash +docker run \ + -v ./init-scripts:/docker-entrypoint-initdb.d:ro \ + -e NEURALDB_PASSWORD=mypassword \ + neuraldb/neuraldb:latest +``` + +```sql +-- init-scripts/01-schema.sql +CREATE TABLE documents ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + content TEXT NOT NULL, + embedding VECTOR(1536), + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() +); + +CREATE INDEX ON documents USING hnsw (embedding vector_cosine_ops); +``` + +## Memory Tuning + +For production, size the container memory based on your dataset: + +``` +Recommended memory = shared_buffers + vector_buffer + (max_connections ร— work_mem) + OS overhead +``` + +For a typical RAG application (5M documents, 1536 dimensions): +- `vector_buffer` โ‰ˆ 5M ร— 1536 ร— 4B ร— 1.3 = ~40 GB +- `shared_buffers` = 8 GB +- `work_mem` ร— connections = 128MB ร— 50 = 6.4 GB +- **Total**: ~56 GB โ€” provision a 64 GB container + +## Upgrading + +```bash +# Pull the new image +docker pull neuraldb/neuraldb:1.1 + +# Stop the current container (data is safe in the volume) +docker stop neuraldb && docker rm neuraldb + +# Start with the new image +docker run -d --name neuraldb \ + -v neuraldb_data:/var/lib/neuraldb/data \ + -e NEURALDB_PASSWORD=mypassword \ + neuraldb/neuraldb:1.1 + +# Run any pending migrations +docker exec neuraldb neuraldb-migrate +``` diff --git a/sample-sites/neuraldb-docs/pages/install-kubernetes.md b/sample-sites/neuraldb-docs/pages/install-kubernetes.md new file mode 100644 index 0000000..2a2c75b --- /dev/null +++ b/sample-sites/neuraldb-docs/pages/install-kubernetes.md @@ -0,0 +1,232 @@ +--- +title: Kubernetes +sort: 110 +section-id: installation +keywords: Kubernetes, Helm, StatefulSet, PVC, k8s, cluster, deployment +description: Deploying NeuralDB on Kubernetes using the official Helm chart and StatefulSets +language: en +--- + +# Kubernetes + +The recommended way to run NeuralDB on Kubernetes is via the official Helm chart. The chart deploys NeuralDB as a StatefulSet with persistent volume claims, and supports both standalone and high-availability configurations. + +## Prerequisites + +- Kubernetes 1.27+ +- Helm 3.x +- A storage class that supports `ReadWriteOnce` PVCs (most cloud providers support this) +- At least 4 CPU cores and 8 GB RAM per NeuralDB node + +## Installing the Helm Chart + +```bash +# Add the NeuralDB Helm repository +helm repo add neuraldb https://charts.neuraldb.io +helm repo update + +# Create a namespace +kubectl create namespace neuraldb + +# Install the chart +helm install neuraldb neuraldb/neuraldb \ + --namespace neuraldb \ + --set auth.password=mysecretpassword \ + --set persistence.size=100Gi +``` + +## Chart Configuration + +Create a `values.yaml` file for production settings: + +```yaml +# values.yaml + +image: + repository: neuraldb/neuraldb + tag: "1.0" + pullPolicy: IfNotPresent + +auth: + # Set via --set auth.password=... or a pre-existing secret + existingSecret: "" + secretKey: "neuraldb-password" + +replicaCount: 1 # primary nodes (use 1 for standalone) +readReplicaCount: 2 # read replicas + +resources: + requests: + cpu: "2" + memory: "8Gi" + limits: + cpu: "8" + memory: "32Gi" + +persistence: + enabled: true + storageClass: "fast-ssd" # use a fast SSD storage class + size: 500Gi + walSize: 50Gi # separate PVC for WAL + +vectorBuffer: "16Gi" # memory for HNSW index +sharedBuffers: "8Gi" # row store page cache +maxConnections: 200 + +service: + type: ClusterIP + port: 5432 + +# High-availability configuration +ha: + enabled: true + replication: + mode: synchronous # 'synchronous' or 'asynchronous' + synchronousCommit: "on" + +backup: + enabled: true + schedule: "0 2 * * *" + s3: + bucket: my-neuraldb-backups + region: us-east-1 + existingSecret: aws-credentials + +monitoring: + enabled: true + serviceMonitor: + enabled: true # requires Prometheus Operator +``` + +Apply the values: + +```bash +helm install neuraldb neuraldb/neuraldb \ + --namespace neuraldb \ + -f values.yaml \ + --set auth.password=$(openssl rand -base64 32) +``` + +## StatefulSet Details + +The chart deploys a `StatefulSet` with: + +- One pod per replica (primary + read replicas) +- Two PVCs per pod: data volume and WAL volume +- An init container that configures replication on startup + +```yaml +# Example pod spec (simplified) +spec: + containers: + - name: neuraldb + image: neuraldb/neuraldb:1.0 + ports: + - containerPort: 5432 + resources: + requests: + memory: "8Gi" + cpu: "2" + volumeMounts: + - name: data + mountPath: /var/lib/neuraldb/data + - name: wal + mountPath: /var/lib/neuraldb/wal + livenessProbe: + exec: + command: ["pg_isready", "-U", "neuraldb"] + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + exec: + command: ["pg_isready", "-U", "neuraldb"] + initialDelaySeconds: 5 + periodSeconds: 5 +``` + +## Services + +The chart creates three Kubernetes services: + +| Service | Type | Port | Description | +|---------|------|------|-------------| +| `neuraldb-primary` | ClusterIP | 5432 | Primary โ€” reads + writes | +| `neuraldb-replica` | ClusterIP | 5432 | Read replicas โ€” reads only | +| `neuraldb-headless` | Headless | 5432 | For StatefulSet pod discovery | + +Connect to the primary: + +```bash +kubectl port-forward svc/neuraldb-primary 5432:5432 -n neuraldb +psql -h localhost -U neuraldb +``` + +## Persistent Volume Claims + +Each pod gets two PVCs: + +```yaml +volumeClaimTemplates: +- metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: fast-ssd + resources: + requests: + storage: 500Gi +- metadata: + name: wal + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: fast-ssd + resources: + requests: + storage: 50Gi +``` + +Use a **fast-ssd** storage class (AWS `gp3`, GCP `pd-ssd`, Azure `Premium_LRS`) for the data and WAL volumes. Spinning disks are not supported in production. + +## Secrets Management + +Store the NeuralDB password in a Kubernetes secret: + +```bash +kubectl create secret generic neuraldb-credentials \ + --namespace neuraldb \ + --from-literal=password=$(openssl rand -base64 32) +``` + +Reference it in `values.yaml`: + +```yaml +auth: + existingSecret: neuraldb-credentials + secretKey: password +``` + +For larger installations, use an external secrets manager (HashiCorp Vault, AWS Secrets Manager) with the External Secrets Operator. + +## Scaling Read Replicas + +Scale the number of read replicas without downtime: + +```bash +helm upgrade neuraldb neuraldb/neuraldb \ + --namespace neuraldb \ + --set readReplicaCount=4 +``` + +The new replica pods will join the replication stream automatically. + +## Upgrading + +```bash +helm repo update +helm upgrade neuraldb neuraldb/neuraldb \ + --namespace neuraldb \ + -f values.yaml \ + --set auth.existingSecret=neuraldb-credentials +``` + +The upgrade performs a rolling update โ€” replicas are updated first, then the primary. diff --git a/sample-sites/neuraldb-docs/pages/install-local.md b/sample-sites/neuraldb-docs/pages/install-local.md new file mode 100644 index 0000000..34be44b --- /dev/null +++ b/sample-sites/neuraldb-docs/pages/install-local.md @@ -0,0 +1,211 @@ +--- +title: Local Development +sort: 130 +section-id: installation +keywords: local, development, binary, homebrew, winget, install, macOS, Linux, Windows +description: Installing NeuralDB locally for development using binaries, Homebrew, or winget +language: en +--- + +# Local Development + +For local development, you can run NeuralDB as a native binary without Docker. This provides lower latency for development workflows and avoids container overhead. + +## System Requirements + +| Platform | Minimum | Recommended | +|----------|---------|-------------| +| macOS | 13.0 (Ventura) | 14.x+ | +| Linux | Ubuntu 22.04 / RHEL 9 | Ubuntu 24.04 | +| Windows | Windows 10 22H2 | Windows 11 | +| CPU | x86-64 or ARM64 | ARM64 (Apple Silicon) | +| RAM | 4 GB | 16 GB+ | +| Disk | 2 GB free | SSD recommended | + +## macOS + +### Homebrew (Recommended) + +```bash +brew tap neuraldb/tap +brew install neuraldb + +# Start as a service (auto-restart on login) +brew services start neuraldb + +# Or start manually (foreground) +neuraldb start + +# Check status +neuraldb status +``` + +The Homebrew formula installs: +- `neuraldb` โ€” the server binary +- `neuraldb-cli` โ€” an enhanced SQL shell (psql-compatible) +- Configuration at `$(brew --prefix)/etc/neuraldb/neuraldb.conf` +- Data directory at `$(brew --prefix)/var/neuraldb` + +### Direct Binary + +```bash +# Apple Silicon (M1/M2/M3/M4) +curl -LO https://releases.neuraldb.io/1.0/neuraldb-macos-arm64.tar.gz +tar -xzf neuraldb-macos-arm64.tar.gz +sudo mv neuraldb /usr/local/bin/ +sudo mv neuraldb-cli /usr/local/bin/ + +# Intel Mac +curl -LO https://releases.neuraldb.io/1.0/neuraldb-macos-amd64.tar.gz +tar -xzf neuraldb-macos-amd64.tar.gz +sudo mv neuraldb /usr/local/bin/ +sudo mv neuraldb-cli /usr/local/bin/ +``` + +## Linux + +### Ubuntu / Debian + +```bash +# Add the NeuralDB APT repository +curl -fsSL https://packages.neuraldb.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/neuraldb-keyring.gpg +echo "deb [signed-by=/usr/share/keyrings/neuraldb-keyring.gpg] https://packages.neuraldb.io/apt stable main" \ + | sudo tee /etc/apt/sources.list.d/neuraldb.list + +sudo apt update +sudo apt install -y neuraldb + +# Start and enable the service +sudo systemctl enable --now neuraldb +``` + +### RHEL / Fedora / CentOS + +```bash +# Add the NeuralDB YUM repository +sudo rpm --import https://packages.neuraldb.io/gpg +sudo tee /etc/yum.repos.d/neuraldb.repo <<'EOF' +[neuraldb] +name=NeuralDB Repository +baseurl=https://packages.neuraldb.io/rpm/stable +enabled=1 +gpgcheck=1 +gpgkey=https://packages.neuraldb.io/gpg +EOF + +sudo dnf install -y neuraldb +sudo systemctl enable --now neuraldb +``` + +### Direct Binary (Linux) + +```bash +# x86-64 +curl -LO https://releases.neuraldb.io/1.0/neuraldb-linux-amd64.tar.gz +tar -xzf neuraldb-linux-amd64.tar.gz +sudo mv neuraldb neuraldb-cli /usr/local/bin/ +``` + +## Windows + +### winget + +```powershell +winget install NeuralDB.NeuralDB +``` + +This installs NeuralDB and registers it as a Windows Service. It starts automatically after installation. + +### Chocolatey + +```powershell +choco install neuraldb +``` + +### MSI Installer + +Download the MSI installer from [neuraldb.io/download](https://neuraldb.io/download) and run it. The installer: +1. Installs `neuraldb.exe` and `neuraldb-cli.exe` to `C:\Program Files\NeuralDB\` +2. Adds them to `PATH` +3. Creates a `NeuralDB` Windows Service +4. Initialises the data directory at `%APPDATA%\NeuralDB\data` + +## First-Time Setup + +After installation, initialise the database: + +```bash +# Linux / macOS +neuraldb init +neuraldb start + +# Connect +neuraldb-cli +# psql prompt: neuraldb=# +``` + +### Change the Default Password + +```sql +ALTER USER neuraldb PASSWORD 'your-new-password'; +``` + +### Create a Development Database + +```sql +CREATE DATABASE myapp; +\c myapp + +CREATE TABLE documents ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + content TEXT, + embedding VECTOR(1536) +); + +CREATE INDEX ON documents USING hnsw (embedding vector_cosine_ops); +``` + +## Configuration + +The default configuration file locations: + +| Platform | Path | +|----------|------| +| macOS (Homebrew) | `$(brew --prefix)/etc/neuraldb/neuraldb.conf` | +| Linux | `/etc/neuraldb/neuraldb.conf` | +| Windows | `%PROGRAMDATA%\NeuralDB\neuraldb.conf` | + +For local development, create a `neuraldb.conf` in your project directory and point to it: + +```bash +neuraldb start --config ./neuraldb.conf +``` + +Useful development settings: + +```ini +# neuraldb.conf (development) +listen_addresses = 'localhost' +port = 5432 +max_connections = 50 +shared_buffers = 256MB +vector_buffer = 1GB +log_min_duration_statement = 100 # log slow queries (>100ms) +log_statement = 'all' # log all SQL (useful for debugging) +``` + +## Uninstalling + +```bash +# macOS +brew services stop neuraldb +brew uninstall neuraldb + +# Ubuntu +sudo systemctl stop neuraldb +sudo apt remove neuraldb + +# Windows +winget uninstall NeuralDB.NeuralDB +# Or: Settings โ†’ Apps โ†’ Installed Apps โ†’ NeuralDB โ†’ Uninstall +``` diff --git a/sample-sites/neuraldb-docs/pages/nql-aggregations.md b/sample-sites/neuraldb-docs/pages/nql-aggregations.md new file mode 100644 index 0000000..2f23a3a --- /dev/null +++ b/sample-sites/neuraldb-docs/pages/nql-aggregations.md @@ -0,0 +1,229 @@ +--- +title: Aggregations +sort: 130 +section-id: query-language +keywords: aggregations, GROUP BY, COUNT, SUM, vectors, AVG, centroid, analytics +description: Aggregating data in NQL including GROUP BY, COUNT, SUM, and vector-specific aggregation functions +language: en +--- + +# Aggregations + +NQL supports the full SQL aggregation toolkit, extended with vector-specific aggregate functions for centroid computation, clustering, and semantic analytics. + +## Standard Aggregations + +All standard SQL aggregate functions work as expected: + +```sql +-- Count documents by category +SELECT category, COUNT(*) AS doc_count +FROM documents +GROUP BY category +ORDER BY doc_count DESC; + +-- Average price by category +SELECT category, + COUNT(*) AS products, + AVG(price) AS avg_price, + MIN(price) AS min_price, + MAX(price) AS max_price, + SUM(stock * price) AS inventory_value +FROM products +WHERE available = true +GROUP BY category +ORDER BY inventory_value DESC; +``` + +## Vector Aggregations + +### `AVG(embedding)` โ€” Centroid Computation + +Compute the centroid (average vector) of a group: + +```sql +-- Centroid of all "technology" documents +SELECT AVG(embedding) AS centroid +FROM documents +WHERE category = 'technology'; +``` + +Use centroids to find documents representative of a cluster: + +```sql +WITH centroid AS ( + SELECT AVG(embedding) AS c FROM documents WHERE category = 'technology' +) +SELECT id, title, 1 - (embedding <=> centroid.c) AS similarity_to_centroid +FROM documents, centroid +WHERE category = 'technology' +ORDER BY embedding <=> centroid.c +LIMIT 10; +``` + +### `vector_centroid(embedding)` โ€” Weighted Centroid + +Compute a weighted centroid using a score column: + +```sql +-- Weighted centroid by rating (higher-rated items pull more) +SELECT vector_centroid(embedding, rating) AS weighted_centroid +FROM products +WHERE category = 'electronics'; +``` + +### `vector_agg_concat(embedding)` โ€” Vector Array + +Collect vectors into an array for downstream processing: + +```sql +SELECT category, vector_agg_concat(embedding) AS all_embeddings +FROM documents +GROUP BY category; +``` + +## GROUP BY with Vector Search + +Find the best document in each category for a given query: + +```sql +SELECT DISTINCT ON (category) + id, category, title, 1 - (embedding <=> $1) AS similarity +FROM documents +WHERE embedding IS NOT NULL +ORDER BY category, embedding <=> $1; +``` + +Or using a lateral join for more control: + +```sql +SELECT cat.category, top_doc.id, top_doc.title, top_doc.similarity +FROM (SELECT DISTINCT category FROM documents) cat, +LATERAL ( + SELECT id, title, 1 - (embedding <=> $1) AS similarity + FROM documents + WHERE category = cat.category + ORDER BY embedding <=> $1 + LIMIT 1 +) top_doc; +``` + +## Window Functions + +Use window functions to rank results within partitions: + +```sql +-- Rank documents by similarity within each category +SELECT + id, title, category, + 1 - (embedding <=> $1) AS similarity, + RANK() OVER ( + PARTITION BY category + ORDER BY embedding <=> $1 + ) AS rank_in_category +FROM documents +WHERE 1 - (embedding <=> $1) > 0.5 +ORDER BY category, rank_in_category; +``` + +Rolling average similarity over time: + +```sql +SELECT + date_trunc('day', created_at) AS day, + AVG(1 - (embedding <=> $1)) AS avg_daily_similarity, + AVG(AVG(1 - (embedding <=> $1))) OVER ( + ORDER BY date_trunc('day', created_at) + ROWS BETWEEN 6 PRECEDING AND CURRENT ROW + ) AS rolling_7d_avg +FROM documents +GROUP BY day +ORDER BY day; +``` + +## Clustering with GROUP BY + +Perform k-means style clustering by assigning documents to their nearest centroid: + +```sql +-- Given pre-computed centroids in a centroids table: +SELECT d.id, d.content, + c.cluster_id, + (d.embedding <=> c.centroid) AS distance_to_centroid +FROM documents d +CROSS JOIN LATERAL ( + SELECT cluster_id, centroid + FROM centroids + ORDER BY d.embedding <=> centroid + LIMIT 1 +) c; +``` + +## HAVING with Vector Conditions + +```sql +-- Categories where the average intra-category similarity is high (tight clusters) +SELECT category, + COUNT(*) AS doc_count, + 1 - AVG(embedding <=> (SELECT AVG(e2.embedding) FROM documents e2 WHERE e2.category = e.category)) AS cohesion +FROM documents e +GROUP BY category +HAVING COUNT(*) > 10 +ORDER BY cohesion DESC; +``` + +## Time-Series Analytics + +Analyse how semantic content shifts over time: + +```sql +-- Daily semantic drift: how different is today's content from last week's? +WITH weekly_centroids AS ( + SELECT + date_trunc('week', created_at) AS week, + AVG(embedding) AS centroid + FROM documents + GROUP BY week +) +SELECT + w1.week, + 1 - (w1.centroid <=> w2.centroid) AS similarity_to_prev_week +FROM weekly_centroids w1 +LEFT JOIN weekly_centroids w2 + ON w2.week = w1.week - INTERVAL '1 week' +ORDER BY w1.week; +``` + +## JSON Aggregation with Vectors + +Combine JSON aggregation with vector results: + +```sql +SELECT + category, + COUNT(*) AS total, + AVG(price) AS avg_price, + JSON_AGG( + JSON_BUILD_OBJECT('id', id, 'name', name, 'similarity', 1 - (embedding <=> $1)) + ORDER BY embedding <=> $1 + ) FILTER (WHERE ROW_NUMBER() OVER (PARTITION BY category ORDER BY embedding <=> $1) <= 3) + AS top_3_per_category +FROM products +WHERE available = true +GROUP BY category; +``` + +## ROLLUP and CUBE + +Standard SQL ROLLUP and CUBE work for hierarchical aggregation: + +```sql +SELECT + region, + category, + COUNT(*) AS count, + AVG(price) AS avg_price +FROM products +GROUP BY ROLLUP(region, category) +ORDER BY region NULLS LAST, category NULLS LAST; +``` diff --git a/sample-sites/neuraldb-docs/pages/nql-basics.md b/sample-sites/neuraldb-docs/pages/nql-basics.md new file mode 100644 index 0000000..53972fc --- /dev/null +++ b/sample-sites/neuraldb-docs/pages/nql-basics.md @@ -0,0 +1,239 @@ +--- +title: NQL Basics +sort: 100 +section-id: query-language +keywords: NQL, NeuralDB Query Language, SQL, syntax, basics, queries +description: Introduction to NeuralDB Query Language (NQL) โ€” syntax, data types, and basic operations +language: en +--- + +# NQL Basics + +NQL (NeuralDB Query Language) is a superset of standard SQL. Every valid SQL statement is also valid NQL. NQL adds extensions for vector operations, embedding generation, and semantic search primitives. + +If you know SQL, you already know most of NQL. This page covers the NQL-specific additions and the data types introduced for AI workloads. + +## Connecting + +NeuralDB speaks the PostgreSQL wire protocol. Connect with any PostgreSQL client: + +```bash +# psql +psql -h localhost -p 5432 -U neuraldb -d mydb + +# neuraldb-cli (enhanced interactive shell) +neuraldb-cli -h localhost +``` + +Connection string format: + +``` +postgresql://[user[:password]@][host][:port][/dbname][?param=value...] +``` + +## Data Types + +### VECTOR(n) + +The core NQL extension. Stores a fixed-length array of 32-bit floats representing a vector embedding: + +```sql +-- Declare a vector column with 1536 dimensions (OpenAI ada-002 output) +CREATE TABLE documents ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + content TEXT NOT NULL, + embedding VECTOR(1536) +); +``` + +Insert a vector by providing a bracketed float array: + +```sql +INSERT INTO documents (content, embedding) +VALUES ('Hello, world', '[0.1, 0.2, 0.3, ...]'); -- 1536 values +``` + +### HALFVEC(n) + +A 16-bit float variant of VECTOR. Half the memory, slight precision loss. Useful when vector_buffer is a constraint: + +```sql +embedding HALFVEC(1536) +``` + +### SPARSEVEC(n) + +Sparse vector representation โ€” stores only non-zero elements. Efficient for high-dimensional but sparse vectors (e.g., BM25 term-frequency vectors): + +```sql +bm25_vector SPARSEVEC(30000) +``` + +## Basic CRUD + +### Creating Tables + +```sql +CREATE TABLE products ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name TEXT NOT NULL, + description TEXT, + category TEXT, + price DECIMAL(10, 2), + stock INTEGER DEFAULT 0, + embedding VECTOR(1536), + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() +); +``` + +### Inserting Data + +```sql +INSERT INTO products (name, description, category, price, stock, embedding) +VALUES ( + 'Wireless Headphones', + 'Premium noise-cancelling wireless headphones with 30-hour battery', + 'electronics', + 299.99, + 150, + '[0.023, -0.187, 0.412, ...]' -- 1536 floats from your embedding model +); +``` + +### Reading Data + +Standard SQL SELECT works as expected: + +```sql +SELECT id, name, price FROM products WHERE category = 'electronics'; +SELECT * FROM products WHERE price BETWEEN 50 AND 300 AND stock > 0; +``` + +### Updating Data + +```sql +UPDATE products SET price = 279.99, embedding = '[...]' WHERE id = $1; +``` + +When updating the `embedding` column, the HNSW index is updated atomically. + +### Deleting Data + +```sql +DELETE FROM products WHERE id = $1; +``` + +## Creating Vector Indexes + +Without an index, vector similarity queries perform exact linear scans (O(n)). Create an HNSW index for sub-linear performance: + +```sql +-- Cosine similarity (most common for text embeddings) +CREATE INDEX ON documents USING hnsw (embedding vector_cosine_ops); + +-- Euclidean distance +CREATE INDEX ON documents USING hnsw (embedding vector_l2_ops); + +-- Dot product (for recommendation systems) +CREATE INDEX ON documents USING hnsw (embedding vector_ip_ops); +``` + +Build an index on an existing large table in parallel: + +```sql +SET max_parallel_maintenance_workers = 8; +CREATE INDEX CONCURRENTLY ON documents USING hnsw (embedding vector_cosine_ops); +``` + +## Basic Vector Queries + +### Find Similar Documents + +```sql +SELECT id, content, 1 - (embedding <=> $1) AS similarity +FROM documents +ORDER BY embedding <=> $1 +LIMIT 10; +``` + +The `<=>` operator is cosine distance (lower = more similar). Subtract from 1 for a similarity score (higher = more similar). + +### Distance Operators + +| Operator | Distance metric | Index ops | +|----------|----------------|-----------| +| `<=>` | Cosine distance | `vector_cosine_ops` | +| `<->` | Euclidean (L2) distance | `vector_l2_ops` | +| `<#>` | Negative dot product | `vector_ip_ops` | + +### Distance Threshold + +```sql +-- Only return results with cosine similarity > 0.8 +SELECT id, content, 1 - (embedding <=> $1) AS similarity +FROM documents +WHERE 1 - (embedding <=> $1) > 0.8 +ORDER BY embedding <=> $1 +LIMIT 20; +``` + +**Note:** The `WHERE 1 - (embedding <=> $1) > 0.8` condition is evaluated after the ANN search, not before. Use `LIMIT` generously enough to capture all relevant results before the threshold filter. + +## NQL Functions + +### `to_vector(text)` + +Convert a string literal to a VECTOR: + +```sql +SELECT to_vector('[0.1, 0.2, 0.3]')::VECTOR(3); +``` + +### `vector_dims(v)` + +Return the number of dimensions: + +```sql +SELECT vector_dims(embedding) FROM documents LIMIT 1; +-- Returns: 1536 +``` + +### `vector_norm(v)` + +Return the L2 norm of a vector: + +```sql +SELECT vector_norm(embedding) FROM documents LIMIT 5; +``` + +### `cosine_similarity(a, b)`, `l2_distance(a, b)`, `dot_product(a, b)` + +Named function alternatives to the operators: + +```sql +SELECT cosine_similarity(embedding, $1) AS similarity +FROM documents +ORDER BY similarity DESC +LIMIT 10; +``` + +## Transactions + +NQL supports full ACID transactions: + +```sql +BEGIN; + +INSERT INTO documents (content, embedding) VALUES ($1, $2); +UPDATE document_stats SET total_count = total_count + 1; + +COMMIT; +``` + +On error, roll back: + +```sql +BEGIN; +-- ... operations ... +ROLLBACK; +``` diff --git a/sample-sites/neuraldb-docs/pages/nql-hybrid.md b/sample-sites/neuraldb-docs/pages/nql-hybrid.md new file mode 100644 index 0000000..f15f2b2 --- /dev/null +++ b/sample-sites/neuraldb-docs/pages/nql-hybrid.md @@ -0,0 +1,216 @@ +--- +title: Hybrid Queries +sort: 120 +section-id: query-language +keywords: hybrid queries, vector, relational, filters, combined, semantic search, metadata +description: Combining vector similarity and relational filters in NQL hybrid queries +language: en +--- + +# Hybrid Queries + +Hybrid queries combine vector similarity search with relational filter predicates in a single SQL statement. The NeuralDB query planner handles the execution strategy โ€” you write normal SQL with vector operators. + +## Basic Hybrid Query + +Find the 10 most semantically similar products that are in the "electronics" category and in stock: + +```sql +SELECT id, name, price, 1 - (embedding <=> $1) AS similarity +FROM products +WHERE category = 'electronics' + AND stock > 0 + AND price < 500 +ORDER BY embedding <=> $1 +LIMIT 10; +``` + +NeuralDB automatically determines whether to: +1. **Pre-filter**: apply relational conditions first, then search the filtered set +2. **Post-filter**: run ANN search, then apply conditions to the top-k results + +The decision is based on the selectivity of the relational predicates. + +## Query Planner Hints + +Override the planner's strategy: + +```sql +-- Force pre-filter (good when relational filter is very selective) +SELECT /*+ PREFILTER */ id, name, score +FROM ( + SELECT id, name, 1 - (embedding <=> $1) AS score + FROM products + WHERE category = 'electronics' -- very selective: 2% of rows +) sub +ORDER BY score DESC +LIMIT 10; + +-- Force post-filter (good when relational filter is weakly selective) +SELECT /*+ POSTFILTER */ id, name, 1 - (embedding <=> $1) AS score +FROM products +WHERE price < 500 -- weakly selective: 80% of rows +ORDER BY embedding <=> $1 +LIMIT 10; +``` + +## Filtering by Multiple Conditions + +```sql +SELECT id, name, description, + 1 - (embedding <=> $1) AS similarity, + price, + rating +FROM products +WHERE category = ANY($2) -- multi-category filter + AND price BETWEEN $3 AND $4 + AND rating >= 4.0 + AND discontinued = false + AND created_at > NOW() - INTERVAL '1 year' +ORDER BY embedding <=> $1 +LIMIT 20; +-- $1 = query embedding +-- $2 = ['electronics', 'computers'] +-- $3 = 50, $4 = 1000 +``` + +## Hybrid Full-Text + Vector (BM25) + +Combine traditional full-text search with vector similarity using Reciprocal Rank Fusion (RRF): + +```sql +WITH vector_search AS ( + SELECT id, ROW_NUMBER() OVER (ORDER BY embedding <=> $1) AS rank + FROM documents + ORDER BY embedding <=> $1 + LIMIT 100 +), +fts_search AS ( + SELECT id, ROW_NUMBER() OVER (ORDER BY ts_rank_cd(tsv, query) DESC) AS rank + FROM documents, to_tsquery('english', $2) query + WHERE tsv @@ query + ORDER BY ts_rank_cd(tsv, query) DESC + LIMIT 100 +), +rrf AS ( + SELECT + COALESCE(v.id, f.id) AS id, + (COALESCE(1.0 / (60 + v.rank), 0) + COALESCE(1.0 / (60 + f.rank), 0)) AS rrf_score + FROM vector_search v + FULL OUTER JOIN fts_search f ON v.id = f.id +) +SELECT d.id, d.content, rrf.rrf_score +FROM rrf +JOIN documents d ON d.id = rrf.id +ORDER BY rrf_score DESC +LIMIT 10; +``` + +NQL also provides a built-in `HYBRID_SEARCH` function: + +```sql +SELECT id, content, score +FROM HYBRID_SEARCH( + table := 'documents', + vector_column := 'embedding', + tsv_column := 'tsv', + query_vector := $1, + query_text := $2, + top_k := 10, + rrf_k := 60, + vector_weight := 0.6, + text_weight := 0.4 +); +``` + +## Joining Vector Results with Other Tables + +```sql +SELECT + p.id, + p.name, + p.price, + c.name AS category_name, + u.display_name AS seller, + 1 - (p.embedding <=> $1) AS similarity +FROM products p +JOIN categories c ON c.id = p.category_id +JOIN users u ON u.id = p.seller_id +WHERE p.available = true + AND c.slug = ANY($2) + AND u.verified = true +ORDER BY p.embedding <=> $1 +LIMIT 15; +``` + +## Subquery Vectors + +Use a subquery to dynamically compute a query vector from existing data: + +```sql +-- Find products similar to product #123 +SELECT id, name, 1 - (embedding <=> ref.embedding) AS similarity +FROM products, + (SELECT embedding FROM products WHERE id = $1) ref +WHERE id != $1 +ORDER BY embedding <=> ref.embedding +LIMIT 10; +``` + +## Tenant-Scoped Search + +For multi-tenant applications, always include tenant filters: + +```sql +SELECT id, content, 1 - (embedding <=> $1) AS similarity +FROM documents +WHERE tenant_id = $2 -- partition pruning if SHARD BY tenant_id + AND embedding IS NOT NULL +ORDER BY embedding <=> $1 +LIMIT 10; +``` + +If the table is sharded by `tenant_id`, this query runs entirely on the correct shard without cross-shard coordination. + +## Composite Scoring + +Combine vector similarity with relational signals: + +```sql +SELECT + id, + name, + price, + rating, + -- Weighted composite score: 70% semantic, 20% rating, 10% recency + (0.7 * (1 - (embedding <=> $1)) + + 0.2 * (rating / 5.0) + + 0.1 * (1 - EXTRACT(DAYS FROM NOW() - created_at) / 365.0) + ) AS composite_score +FROM products +WHERE available = true + AND price < $2 +ORDER BY composite_score DESC +LIMIT 20; +``` + +## Pagination + +Cursor-based pagination for vector results: + +```sql +-- Page 1 +SELECT id, name, (embedding <=> $1) AS dist +FROM products +WHERE available = true +ORDER BY dist, id -- secondary sort by id for stable pagination +LIMIT 20; + +-- Page 2 (cursor: last dist and id from page 1) +SELECT id, name, (embedding <=> $1) AS dist +FROM products +WHERE available = true + AND ((embedding <=> $1) > $2 OR ((embedding <=> $1) = $2 AND id > $3)) +ORDER BY dist, id +LIMIT 20; +``` diff --git a/sample-sites/neuraldb-docs/pages/nql-transactions.md b/sample-sites/neuraldb-docs/pages/nql-transactions.md new file mode 100644 index 0000000..d91af8a --- /dev/null +++ b/sample-sites/neuraldb-docs/pages/nql-transactions.md @@ -0,0 +1,237 @@ +--- +title: Transactions +sort: 140 +section-id: query-language +keywords: transactions, ACID, isolation levels, MVCC, BEGIN, COMMIT, ROLLBACK +description: ACID transactions in NeuralDB โ€” isolation levels, MVCC, savepoints, and advisory locks +language: en +--- + +# Transactions + +NeuralDB provides full ACID transactions with MVCC (Multi-Version Concurrency Control). Unlike most vector databases, NeuralDB guarantees atomicity across both relational and vector data within a single transaction. + +## ACID Guarantees + +| Property | Guarantee | +|----------|---------| +| **Atomicity** | All operations in a transaction succeed or all are rolled back โ€” including vector index updates | +| **Consistency** | Constraints (foreign keys, unique indexes, not null) are enforced at commit time | +| **Isolation** | Concurrent transactions do not see each other's uncommitted changes | +| **Durability** | Committed transactions survive crashes via the WAL | + +## Basic Transaction Syntax + +```sql +BEGIN; + +-- Your operations here +INSERT INTO documents (content, embedding) VALUES ($1, $2); +UPDATE document_stats SET total_count = total_count + 1; +INSERT INTO audit_log (action, data) VALUES ('insert', $3); + +COMMIT; +``` + +On error, roll back: + +```sql +BEGIN; + +INSERT INTO documents (content, embedding) VALUES ($1, $2); + +-- Something went wrong +ROLLBACK; +``` + +## Isolation Levels + +NeuralDB supports four isolation levels. Set them with `SET TRANSACTION ISOLATION LEVEL`: + +```sql +BEGIN; +SET TRANSACTION ISOLATION LEVEL READ COMMITTED; +-- ... your queries ... +COMMIT; +``` + +### Read Committed (Default) + +Each statement sees only rows committed before that statement began. Two successive reads within the same transaction may see different data if another transaction commits between them. + +```sql +BEGIN; +-- Sees all rows committed before this SELECT +SELECT COUNT(*) FROM documents; -- Returns 1000 + +-- Another transaction inserts and commits a row here + +-- Sees the new row (non-repeatable read) +SELECT COUNT(*) FROM documents; -- Returns 1001 +COMMIT; +``` + +### Repeatable Read + +A transaction sees only rows committed before the transaction began. Reads are stable throughout the transaction. + +```sql +BEGIN ISOLATION LEVEL REPEATABLE READ; +SELECT COUNT(*) FROM documents; -- Returns 1000 + +-- Another transaction inserts and commits + +SELECT COUNT(*) FROM documents; -- Still 1000 โ€” repeatable read +COMMIT; +``` + +### Serializable + +The strictest level. Transactions execute as if they ran serially one after another. NeuralDB uses Serializable Snapshot Isolation (SSI) โ€” it allows concurrent execution but detects and aborts transactions that would produce a non-serializable outcome. + +```sql +BEGIN ISOLATION LEVEL SERIALIZABLE; +-- ... complex read-modify-write patterns ... +COMMIT; +-- May raise: ERROR: could not serialize access โ€” retry the transaction +``` + +### Read Uncommitted + +NeuralDB maps this to Read Committed (it does not implement dirty reads). + +## Retry Logic + +Serializable transactions can fail with serialization errors. Always retry: + +```python +from psycopg2 import errors + +MAX_RETRIES = 5 + +for attempt in range(MAX_RETRIES): + try: + with conn.cursor() as cur: + cur.execute("BEGIN ISOLATION LEVEL SERIALIZABLE") + # ... your operations ... + cur.execute("COMMIT") + break + except errors.SerializationFailure: + conn.rollback() + if attempt == MAX_RETRIES - 1: + raise + time.sleep(0.1 * (2 ** attempt)) # exponential backoff +``` + +## Savepoints + +Savepoints allow partial rollbacks within a transaction: + +```sql +BEGIN; + +INSERT INTO documents (content, embedding) VALUES ($1, $2); + +SAVEPOINT after_insert; + +-- Risky operation +UPDATE document_stats SET count = count + 1 WHERE id = $3; + +-- Oh no, something went wrong โ€” roll back to the savepoint +ROLLBACK TO SAVEPOINT after_insert; + +-- The INSERT is still pending โ€” we can try a different approach +UPDATE document_stats SET count = count + 1 WHERE id = $4; + +COMMIT; +``` + +## Vector Transactions + +Vector index updates are transactional in NeuralDB. An HNSW index entry is added atomically with the row: + +```sql +BEGIN; + +-- Both the row and the vector index entry are inserted atomically +INSERT INTO documents (id, content, embedding) VALUES ($1, $2, $3); + +-- If we ROLLBACK, neither the row nor the index entry will exist +ROLLBACK; + +-- After rollback, a similarity search will NOT find $1 +SELECT id FROM documents ORDER BY embedding <=> $3 LIMIT 1; +-- $1 is not returned +``` + +## Long-Running Transactions + +Avoid long-running transactions โ€” they: +- Hold row-level locks, blocking other writes +- Prevent VACUUM from reclaiming dead rows (bloat) +- Increase the risk of deadlocks + +Set a statement timeout to kill runaway queries: + +```sql +SET statement_timeout = '30s'; +``` + +Set a transaction timeout: + +```sql +SET idle_in_transaction_session_timeout = '5min'; +``` + +## Deadlock Detection + +NeuralDB automatically detects deadlocks and aborts one of the transactions: + +``` +ERROR: deadlock detected +DETAIL: Process 12345 waits for ShareLock on transaction 67890; blocked by process 99999. +Hint: See server log for query details. +``` + +Minimise deadlock risk by always acquiring locks in the same order across all transactions. + +## Advisory Locks + +For application-level locking (e.g., ensuring only one worker processes a job): + +```sql +-- Acquire a session-level advisory lock (blocks until acquired) +SELECT pg_advisory_lock(42); + +-- Try to acquire (returns false if already held) +SELECT pg_try_advisory_lock(42); -- returns boolean + +-- Release +SELECT pg_advisory_unlock(42); + +-- Transaction-level (auto-released at commit/rollback) +SELECT pg_advisory_xact_lock(42); +``` + +## Two-Phase Commit (2PC) + +For distributed transactions spanning multiple systems: + +```sql +-- Phase 1: Prepare +BEGIN; +-- ... operations ... +PREPARE TRANSACTION 'my-distributed-txn-id'; + +-- Phase 2: Commit or rollback +COMMIT PREPARED 'my-distributed-txn-id'; +-- or +ROLLBACK PREPARED 'my-distributed-txn-id'; +``` + +Check pending prepared transactions: + +```sql +SELECT gid, prepared, owner, database +FROM pg_prepared_xacts; +``` diff --git a/sample-sites/neuraldb-docs/pages/nql-vectors.md b/sample-sites/neuraldb-docs/pages/nql-vectors.md new file mode 100644 index 0000000..22d4fbf --- /dev/null +++ b/sample-sites/neuraldb-docs/pages/nql-vectors.md @@ -0,0 +1,215 @@ +--- +title: Vector Queries +sort: 110 +section-id: query-language +keywords: vector queries, NEAREST, SIMILAR, cosine, dot product, euclidean, ANN +description: Writing vector similarity queries in NQL โ€” NEAREST, SIMILAR, distance operators, and recall tuning +language: en +--- + +# Vector Queries + +NQL extends standard SQL with operators and functions for vector similarity search. This page covers every method for querying vectors, from basic nearest-neighbour lookups to advanced recall tuning. + +## Distance Operators + +NQL provides three distance operators that double as index-acceleration hints: + +```sql +-- Cosine distance (returns 0 to 2, lower = more similar) +embedding <=> query_vector + +-- Euclidean (L2) distance (returns 0 to โˆž, lower = more similar) +embedding <-> query_vector + +-- Negative dot product (higher inner product = more similar โ†’ negate for ORDER BY) +embedding <#> query_vector +``` + +Always pair `ORDER BY` with `LIMIT` when using distance operators โ€” the planner uses the HNSW index only when there is an explicit `ORDER BY ... LIMIT`: + +```sql +-- โœ… Uses HNSW index +SELECT id, content FROM documents +ORDER BY embedding <=> '[0.1, 0.2, ...]' +LIMIT 10; + +-- โŒ Full scan (no ORDER BY ... LIMIT) +SELECT id, content FROM documents +WHERE (embedding <=> '[0.1, 0.2, ...]') < 0.3; +``` + +## NEAREST Clause + +NQL provides a syntactic alternative to `ORDER BY ... LIMIT` for nearest-neighbour queries: + +```sql +SELECT id, content, score +FROM documents +NEAREST TO embedding = '[0.1, 0.2, ...]' USING COSINE +TOP 10; +``` + +This is equivalent to: + +```sql +SELECT id, content, 1 - (embedding <=> '[0.1, 0.2, ...]') AS score +FROM documents +ORDER BY embedding <=> '[0.1, 0.2, ...]' +LIMIT 10; +``` + +The `NEAREST TO` clause is more readable and allows NeuralDB to apply additional optimisations. + +### Distance Metrics in NEAREST + +```sql +NEAREST TO embedding = $1 USING COSINE TOP 10 +NEAREST TO embedding = $1 USING EUCLIDEAN TOP 10 +NEAREST TO embedding = $1 USING DOT_PRODUCT TOP 10 +``` + +## SIMILAR Clause + +`SIMILAR` returns results above a similarity threshold rather than a fixed count. Because the threshold is checked after the ANN search, NeuralDB must retrieve an initial candidate set. Use `LIMIT` to cap the candidates: + +```sql +SELECT id, content, score +FROM documents +SIMILAR TO embedding = $1 USING COSINE THRESHOLD 0.75 +LIMIT 100; +``` + +This returns up to 100 documents with cosine similarity โ‰ฅ 0.75. + +## Returning Scores + +Include the distance or similarity score in results: + +```sql +-- Distance (lower = more similar) +SELECT id, content, (embedding <=> $1) AS distance +FROM documents +ORDER BY embedding <=> $1 +LIMIT 10; + +-- Similarity (higher = more similar, cosine) +SELECT id, content, 1 - (embedding <=> $1) AS similarity +FROM documents +ORDER BY embedding <=> $1 +LIMIT 10; +``` + +## Querying With a Vector Literal + +Pass vectors as SQL parameters (recommended) or literals: + +```sql +-- Parameterised (prevents injection, preferred) +SELECT id, content FROM documents +ORDER BY embedding <=> $1 +LIMIT 10; +-- $1 = '[0.023, -0.187, 0.412, ...]' + +-- Inline literal (useful in SQL shells) +SELECT id, content FROM documents +ORDER BY embedding <=> '[0.023, -0.187, 0.412]'::VECTOR(3) +LIMIT 5; +``` + +## Recall Tuning + +The HNSW index trades recall for performance. By default, `hnsw.ef_search = 40`, which provides ~95% recall at ~1ms latency for 10M vectors. + +Increase `ef_search` for higher recall: + +```sql +-- Set for the current session +SET hnsw.ef_search = 200; + +-- Set for the current transaction +BEGIN; +SET LOCAL hnsw.ef_search = 200; +SELECT * FROM documents ORDER BY embedding <=> $1 LIMIT 10; +COMMIT; +``` + +Typical recall vs performance trade-off (10M 1536-dim vectors, 32 vCPU): + +| ef_search | Recall@10 | p50 latency | QPS | +|-----------|-----------|-------------|-----| +| 20 | 89% | 0.7ms | 12,000 | +| 40 | 95% | 1.2ms | 8,400 | +| 80 | 98% | 2.1ms | 4,800 | +| 200 | 99.5% | 4.8ms | 2,100 | +| exact | 100% | 45ms | 220 | + +## Exact Search + +Force exact (brute-force) nearest-neighbour search, ignoring the HNSW index: + +```sql +SET neuraldb.vector_scan = 'exact'; +SELECT * FROM documents ORDER BY embedding <=> $1 LIMIT 10; +RESET neuraldb.vector_scan; +``` + +Use exact search when: +- You need 100% recall (e.g., de-duplication, exact compliance checks) +- The table has fewer than ~100k rows (exact is competitive) +- You are benchmarking ANN recall + +## Bulk Vector Operations + +### Batch Insert + +Use `COPY` for high-throughput ingestion: + +```bash +# Format: id\tcontent\tembedding +psql -c "\COPY documents (id, content, embedding) FROM '/data/vectors.tsv'" +``` + +### Updating Embeddings in Bulk + +```sql +UPDATE documents +SET embedding = new_embeddings.embedding +FROM (VALUES + ('uuid-1', '[...]'::VECTOR(1536)), + ('uuid-2', '[...]'::VECTOR(1536)) +) AS new_embeddings(id, embedding) +WHERE documents.id = new_embeddings.id::UUID; +``` + +## Multi-Vector Queries + +Find documents closest to ANY of multiple query vectors (OR semantics): + +```sql +WITH queries AS ( + SELECT UNNEST(ARRAY['[...]'::VECTOR(1536), '[...]'::VECTOR(1536)]) AS qv +), +ranked AS ( + SELECT d.id, d.content, MIN(d.embedding <=> q.qv) AS best_distance + FROM documents d, queries q + GROUP BY d.id, d.content +) +SELECT * FROM ranked +ORDER BY best_distance +LIMIT 20; +``` + +## Vector Arithmetic + +NQL supports vector arithmetic for query expansion and centroid computation: + +```sql +-- Average embedding of a result set (cluster centroid) +SELECT AVG(embedding) FROM documents WHERE category = 'technology'; + +-- Find documents similar to the average +SELECT id, content FROM documents +ORDER BY embedding <=> (SELECT AVG(embedding) FROM documents WHERE category = 'tech') +LIMIT 10; +``` diff --git a/sample-sites/neuraldb-docs/pages/ops-backup.md b/sample-sites/neuraldb-docs/pages/ops-backup.md new file mode 100644 index 0000000..7bd2189 --- /dev/null +++ b/sample-sites/neuraldb-docs/pages/ops-backup.md @@ -0,0 +1,227 @@ +--- +title: Backup & Restore +sort: 110 +section-id: operations +keywords: backup, restore, snapshot, WAL archiving, PITR, point-in-time recovery +description: Backup and restore strategies for NeuralDB โ€” snapshots, WAL archiving, and point-in-time recovery +language: en +--- + +# Backup & Restore + +A comprehensive backup strategy for NeuralDB combines base snapshots with continuous WAL archiving, enabling point-in-time recovery (PITR) to any moment within your retention window. + +## Backup Strategies + +| Strategy | Recovery point objective | Recovery time | Storage | +|----------|--------------------------|---------------|---------| +| Snapshot only | Time of last snapshot | Fast | Medium | +| WAL archiving only | Continuous (any point) | Slow | High | +| Snapshot + WAL | Best of both | Fast | High | + +**Recommendation:** Use snapshot + WAL archiving in production. Take daily base snapshots and archive WAL continuously. + +## Physical Snapshot (pg_basebackup) + +`pg_basebackup` creates a consistent physical copy of the data directory: + +```bash +# Full backup โ€” local filesystem +pg_basebackup \ + --host=localhost \ + --port=5432 \ + --username=backup_user \ + --pgdata=/backups/neuraldb/$(date +%Y%m%d) \ + --wal-method=stream \ + --checkpoint=fast \ + --compress=lz4 \ + --progress \ + --verbose + +# Full backup โ€” tar format (smaller, easier to upload to S3) +pg_basebackup \ + --host=localhost \ + --pgdata=- \ + --format=tar \ + --wal-method=stream \ + --compress=lz4 \ + | aws s3 cp - s3://my-backups/neuraldb/base-$(date +%Y%m%d).tar.lz4 +``` + +Create a dedicated backup user: + +```sql +CREATE USER backup_user WITH REPLICATION PASSWORD 'backup-password'; +GRANT CONNECT ON DATABASE neuraldb TO backup_user; +``` + +## WAL Archiving + +WAL archiving copies each WAL segment to a secure location as it is completed. Combined with a base snapshot, this enables PITR. + +Enable WAL archiving: + +```ini +# neuraldb.conf +wal_level = replica +archive_mode = on +archive_command = 'aws s3 cp %p s3://my-backups/neuraldb/wal/%f' +archive_timeout = 60 # archive at least every 60 seconds even if no WAL activity +``` + +Verify archiving is working: + +```sql +SELECT last_archived_wal, last_archived_time, + last_failed_wal, last_failed_time, + archived_count, failed_count +FROM pg_stat_archiver; +``` + +### S3 Archive Command + +```bash +#!/bin/bash +# /usr/local/bin/neuraldb-archive.sh +# Usage: %p = source file path, %f = file name + +set -e +SOURCE="$1" +DEST_FILE="$2" +S3_BUCKET="${ARCHIVE_S3_BUCKET}" +S3_PREFIX="${ARCHIVE_S3_PREFIX:-neuraldb/wal/}" + +aws s3 cp "$SOURCE" "s3://${S3_BUCKET}/${S3_PREFIX}${DEST_FILE}" \ + --storage-class STANDARD_IA \ + --sse aws:kms +``` + +```ini +archive_command = '/usr/local/bin/neuraldb-archive.sh %p %f' +``` + +## Automated Backups with pgBackRest + +pgBackRest is the recommended tool for production NeuralDB backups: + +```bash +# Install +sudo apt install pgbackrest + +# Configure +sudo tee /etc/pgbackrest/pgbackrest.conf <<'EOF' +[global] +repo1-path=/var/lib/pgbackrest +repo1-retention-full=7 +repo1-retention-diff=14 +repo1-type=s3 +repo1-s3-bucket=my-neuraldb-backups +repo1-s3-endpoint=s3.amazonaws.com +repo1-s3-region=us-east-1 +compress-type=lz4 +start-fast=y +backup-standby=y + +[neuraldb] +pg1-path=/var/lib/neuraldb/data +pg1-port=5432 +pg1-user=backup_user +EOF + +# Initialise +sudo -u postgres pgbackrest --stanza=neuraldb stanza-create + +# Full backup +sudo -u postgres pgbackrest --stanza=neuraldb backup --type=full + +# Differential backup (only changes since last full) +sudo -u postgres pgbackrest --stanza=neuraldb backup --type=diff + +# Incremental (only changes since last backup of any type) +sudo -u postgres pgbackrest --stanza=neuraldb backup --type=incr +``` + +Schedule backups with cron: + +```cron +# /etc/cron.d/neuraldb-backup +0 1 * * 0 postgres pgbackrest --stanza=neuraldb backup --type=full +0 1 * * 1-6 postgres pgbackrest --stanza=neuraldb backup --type=diff +``` + +## Point-in-Time Recovery (PITR) + +To restore to a specific point in time: + +```bash +# Stop NeuralDB +systemctl stop neuraldb + +# Restore a base backup +pgbackrest --stanza=neuraldb restore \ + --target="2026-05-15 14:30:00+00" \ + --target-action=promote \ + --delta + +# Or restore to just before a specific transaction +pgbackrest --stanza=neuraldb restore \ + --target-name="before_accidental_delete" \ + --target-action=promote + +# Start NeuralDB โ€” it will replay WAL up to the target point +systemctl start neuraldb +``` + +Create named restore points before risky operations: + +```sql +-- Before running a migration +SELECT pg_create_restore_point('before_migration_20260515'); +``` + +## Logical Backup (pg_dump) + +For smaller databases or table-level backups, `pg_dump` provides a logical backup: + +```bash +# Dump entire database +pg_dump -h localhost -U neuraldb mydb | \ + lz4 | \ + aws s3 cp - s3://my-backups/neuraldb/logical-$(date +%Y%m%d).sql.lz4 + +# Dump specific table +pg_dump -h localhost -U neuraldb -t documents mydb > documents-backup.sql + +# Dump in custom format (best compression, selective restore) +pg_dump -Fc -h localhost -U neuraldb mydb > mydb-$(date +%Y%m%d).dump +``` + +**Note:** Logical backups do not include vector index data โ€” only the raw vector column values. After restore, recreate indexes manually. + +## Restoring from pg_dump + +```bash +# Restore entire database +lz4 -d backup.sql.lz4 | psql -h localhost -U neuraldb -d mydb_restore + +# Restore custom format +pg_restore -h localhost -U neuraldb -d mydb_restore --jobs=8 mydb.dump + +# Restore a single table +pg_restore -h localhost -U neuraldb -d mydb -t documents mydb.dump +``` + +## Testing Backups + +Never trust backups you haven't tested. Automate monthly restore tests: + +```bash +#!/bin/bash +# Test backup restore in a separate environment +pgbackrest --stanza=neuraldb restore --pg1-path=/tmp/restore-test --delta +pg_ctl -D /tmp/restore-test start +psql -h /tmp/restore-test -c "SELECT COUNT(*) FROM documents;" neuraldb +pg_ctl -D /tmp/restore-test stop +rm -rf /tmp/restore-test +echo "Restore test passed: $(date)" +``` diff --git a/sample-sites/neuraldb-docs/pages/ops-migration.md b/sample-sites/neuraldb-docs/pages/ops-migration.md new file mode 100644 index 0000000..f8a6e42 --- /dev/null +++ b/sample-sites/neuraldb-docs/pages/ops-migration.md @@ -0,0 +1,250 @@ +--- +title: Migration +sort: 130 +section-id: operations +keywords: migration, import, Postgres, Pinecone, Weaviate, data migration, ETL +description: Migrating data to NeuralDB from PostgreSQL, Pinecone, Weaviate, and other sources +language: en +--- + +# Migration + +This guide covers migrating data into NeuralDB from common sources: PostgreSQL (with or without pgvector), Pinecone, and Weaviate. + +## From PostgreSQL (without vectors) + +If you are migrating a standard PostgreSQL database to NeuralDB, the simplest path is a logical dump and restore: + +```bash +# 1. Dump from source Postgres +pg_dump \ + -h source-host \ + -U source-user \ + -d source-database \ + --format=custom \ + --compress=9 \ + > source-backup.dump + +# 2. Create the target database in NeuralDB +psql -h neuraldb-host -U neuraldb -c "CREATE DATABASE myapp;" + +# 3. Restore into NeuralDB +pg_restore \ + -h neuraldb-host \ + -U neuraldb \ + -d myapp \ + --jobs=8 \ + --no-owner \ + source-backup.dump +``` + +### Adding Vector Columns Post-Migration + +After restoring the schema and data, add vector columns and generate embeddings: + +```sql +-- Add the vector column +ALTER TABLE documents ADD COLUMN embedding VECTOR(1536); + +-- Create the index (do this before backfilling on large tables) +CREATE INDEX CONCURRENTLY documents_embedding_idx +ON documents USING hnsw (embedding vector_cosine_ops); +``` + +Then backfill embeddings in batches: + +```python +import openai +from neuraldb import NeuralDB + +client = NeuralDB(connection_string) +openai_client = openai.OpenAI() + +BATCH_SIZE = 100 + +while True: + rows = client.query(""" + SELECT id, content FROM documents + WHERE embedding IS NULL + LIMIT %s + """, [BATCH_SIZE]) + + if not rows: + break + + texts = [row['content'] for row in rows] + response = openai_client.embeddings.create( + model="text-embedding-3-small", + input=texts + ) + + updates = [ + (response.data[i].embedding, rows[i]['id']) + for i in range(len(rows)) + ] + + client.executemany( + "UPDATE documents SET embedding = %s WHERE id = %s", + updates + ) + print(f"Backfilled {len(rows)} rows") +``` + +## From PostgreSQL + pgvector + +pgvector uses the same `VECTOR` type as NeuralDB. Migration is a direct dump and restore with minimal adjustments. + +```bash +# Dump โ€” exclude pgvector extension (NeuralDB has native vector support) +pg_dump \ + -h source-host -U source-user -d source-db \ + --format=custom \ + --exclude-extension=vector \ + > pgvector-backup.dump + +pg_restore \ + -h neuraldb-host -U neuraldb -d myapp \ + --jobs=8 \ + pgvector-backup.dump +``` + +### Re-create HNSW Indexes + +pgvector HNSW indexes are not transferred. Recreate them in NeuralDB: + +```sql +-- Drop pgvector-created indexes +DROP INDEX IF EXISTS documents_embedding_idx; + +-- Create NeuralDB HNSW index (same syntax, better performance) +CREATE INDEX CONCURRENTLY documents_embedding_idx +ON documents USING hnsw (embedding vector_cosine_ops) +WITH (m = 16, ef_construction = 64); +``` + +## From Pinecone + +Pinecone stores vectors with metadata. Export using the Pinecone SDK and ingest into NeuralDB: + +```python +import pinecone +from neuraldb import NeuralDB, BulkIngestor + +# Source: Pinecone +pc = pinecone.Pinecone(api_key=os.environ["PINECONE_API_KEY"]) +index = pc.Index("my-index") + +# Target: NeuralDB +client = NeuralDB(os.environ["NEURALDB_URL"]) + +# Create target table +client.execute(""" + CREATE TABLE IF NOT EXISTS pinecone_migration ( + id TEXT PRIMARY KEY, + embedding VECTOR(1536), + metadata JSONB, + migrated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() + ) +""") + +client.execute(""" + CREATE INDEX IF NOT EXISTS pinecone_migration_emb_idx + ON pinecone_migration USING hnsw (embedding vector_cosine_ops) +""") + +# Paginate through all Pinecone vectors +ingestor = BulkIngestor(client, table="pinecone_migration", batch_size=500) + +with ingestor as ing: + for ids_batch in paginate_pinecone_ids(index, batch_size=1000): + fetch_response = index.fetch(ids=ids_batch) + + for vector_id, vector_data in fetch_response.vectors.items(): + ing.add({ + "id": vector_id, + "embedding": vector_data.values, + "metadata": vector_data.metadata or {} + }) + +print(f"Migrated {ingestor.total_inserted} vectors") +``` + +### Mapping Pinecone Metadata to Columns + +Flatten commonly-queried metadata fields into dedicated columns for better query performance: + +```python +# Instead of: metadata JSONB +# Create typed columns for common filter fields: +client.execute(""" + ALTER TABLE pinecone_migration + ADD COLUMN IF NOT EXISTS category TEXT GENERATED ALWAYS AS (metadata->>'category') STORED, + ADD COLUMN IF NOT EXISTS created_date DATE GENERATED ALWAYS AS ((metadata->>'date')::DATE) STORED; + + CREATE INDEX ON pinecone_migration (category); + CREATE INDEX ON pinecone_migration (created_date); +""") +``` + +## From Weaviate + +Export Weaviate data using the Weaviate client SDK: + +```python +import weaviate +from neuraldb import NeuralDB, BulkIngestor + +weaviate_client = weaviate.connect_to_local() +neuraldb_client = NeuralDB(os.environ["NEURALDB_URL"]) + +collection = weaviate_client.collections.get("Document") + +# Create target schema +neuraldb_client.execute(""" + CREATE TABLE weaviate_documents ( + id UUID PRIMARY KEY, + content TEXT, + category TEXT, + source TEXT, + embedding VECTOR(1536) + ); + CREATE INDEX ON weaviate_documents USING hnsw (embedding vector_cosine_ops); +""") + +ingestor = BulkIngestor(neuraldb_client, table="weaviate_documents", batch_size=500) + +with ingestor as ing: + for item in collection.iterator(include_vector=True): + ing.add({ + "id": str(item.uuid), + "content": item.properties.get("content", ""), + "category": item.properties.get("category"), + "source": item.properties.get("source"), + "embedding": item.vector.get("default"), + }) + +weaviate_client.close() +print(f"Migrated {ingestor.total_inserted} objects") +``` + +## Verifying Migration + +After any migration, verify data integrity: + +```sql +-- Row count comparison +SELECT COUNT(*) FROM documents; + +-- Sample vector similarity (should match source) +SELECT id, content, 1 - (embedding <=> (SELECT embedding FROM documents LIMIT 1)) AS sim +FROM documents +ORDER BY embedding <=> (SELECT embedding FROM documents LIMIT 1) +LIMIT 5; + +-- Check for null embeddings +SELECT COUNT(*) FROM documents WHERE embedding IS NULL; + +-- Index health +SELECT index_name, hnsw_in_memory, estimated_recall +FROM neuraldb_stat_vector_indexes; +``` diff --git a/sample-sites/neuraldb-docs/pages/ops-monitoring.md b/sample-sites/neuraldb-docs/pages/ops-monitoring.md new file mode 100644 index 0000000..3838a6b --- /dev/null +++ b/sample-sites/neuraldb-docs/pages/ops-monitoring.md @@ -0,0 +1,214 @@ +--- +title: Monitoring +sort: 100 +section-id: operations +keywords: monitoring, Prometheus, Grafana, metrics, alerts, observability, dashboards +description: Monitoring NeuralDB with Prometheus metrics, Grafana dashboards, and alert configuration +language: en +--- + +# Monitoring + +![NeuralDB Dashboard](assets/images/dashboard.jpg) + +Observability is critical for database operations. NeuralDB exposes Prometheus-compatible metrics and provides an official Grafana dashboard for real-time monitoring. + +## Prometheus Metrics + +NeuralDB exposes metrics at `http://localhost:9187/metrics` (via the bundled exporter). + +Enable the metrics exporter: + +```ini +# neuraldb.conf +metrics.enabled = true +metrics.port = 9187 +metrics.path = /metrics +``` + +Or run the standalone exporter: + +```bash +neuraldb_exporter \ + --web.listen-address=:9187 \ + --db.uri="postgresql://monitor:password@localhost:5432/neuraldb?sslmode=disable" +``` + +### Key Metrics + +#### Connection Metrics + +| Metric | Type | Description | +|--------|------|-------------| +| `neuraldb_connections_total` | Gauge | Current connections by state | +| `neuraldb_connections_max` | Gauge | `max_connections` setting | +| `neuraldb_connection_pool_waiting` | Gauge | Queries waiting for a connection | + +#### Query Metrics + +| Metric | Type | Description | +|--------|------|-------------| +| `neuraldb_queries_total` | Counter | Total queries by database and status | +| `neuraldb_query_duration_seconds` | Histogram | Query duration (p50, p95, p99) | +| `neuraldb_slow_queries_total` | Counter | Queries exceeding `log_min_duration_statement` | +| `neuraldb_deadlocks_total` | Counter | Deadlocks detected | + +#### Vector Metrics + +| Metric | Type | Description | +|--------|------|-------------| +| `neuraldb_vector_queries_total` | Counter | Vector similarity queries by index | +| `neuraldb_vector_query_duration_seconds` | Histogram | ANN query latency | +| `neuraldb_hnsw_index_size_bytes` | Gauge | In-memory size of HNSW graphs | +| `neuraldb_hnsw_build_duration_seconds` | Histogram | Time to build HNSW indexes | +| `neuraldb_vector_recall_ratio` | Gauge | Estimated recall for ANN queries | + +#### Replication Metrics + +| Metric | Type | Description | +|--------|------|-------------| +| `neuraldb_replication_lag_bytes` | Gauge | WAL lag per replica | +| `neuraldb_replication_lag_seconds` | Gauge | Time lag per replica | +| `neuraldb_wal_size_bytes` | Gauge | Current WAL on-disk size | + +#### Storage Metrics + +| Metric | Type | Description | +|--------|------|-------------| +| `neuraldb_database_size_bytes` | Gauge | Total database size | +| `neuraldb_table_size_bytes` | Gauge | Size per table | +| `neuraldb_bloat_ratio` | Gauge | Estimated dead row ratio | +| `neuraldb_checkpoint_duration_seconds` | Histogram | Checkpoint write time | + +## Prometheus Configuration + +```yaml +# prometheus.yml +scrape_configs: + - job_name: 'neuraldb' + static_configs: + - targets: ['localhost:9187'] + scrape_interval: 15s + metrics_path: /metrics +``` + +## Grafana Dashboard + +Import the official NeuralDB dashboard from Grafana.com (Dashboard ID: **18921**): + +```bash +# Import via Grafana API +curl -X POST \ + http://admin:password@localhost:3000/api/dashboards/import \ + -H "Content-Type: application/json" \ + -d '{ "gnetId": 18921, "overwrite": true, "inputs": [{"name": "DS_PROMETHEUS", "type": "datasource", "pluginId": "prometheus", "value": "Prometheus"}] }' +``` + +The dashboard includes panels for: +- Query rate and error rate +- Query latency percentiles (p50, p95, p99) +- Active connections vs max connections +- Vector index memory usage +- Replication lag +- Database and table sizes +- Cache hit ratio +- Checkpoint frequency + +## Alerting Rules + +Create Prometheus alerting rules for critical conditions: + +```yaml +# neuraldb-alerts.yml +groups: + - name: neuraldb + rules: + + - alert: NeuralDBConnectionsHigh + expr: neuraldb_connections_total{state="active"} / neuraldb_connections_max > 0.85 + for: 2m + labels: + severity: warning + annotations: + summary: "NeuralDB connections above 85%" + description: "{{ $value | humanizePercentage }} of max connections in use" + + - alert: NeuralDBConnectionsExhausted + expr: neuraldb_connections_total{state="active"} / neuraldb_connections_max > 0.98 + for: 30s + labels: + severity: critical + annotations: + summary: "NeuralDB connections nearly exhausted" + + - alert: NeuralDBHighQueryLatency + expr: histogram_quantile(0.99, rate(neuraldb_query_duration_seconds_bucket[5m])) > 1.0 + for: 5m + labels: + severity: warning + annotations: + summary: "P99 query latency above 1 second" + + - alert: NeuralDBReplicationLagHigh + expr: neuraldb_replication_lag_seconds > 30 + for: 1m + labels: + severity: warning + annotations: + summary: "Replication lag above 30 seconds" + + - alert: NeuralDBDiskSpaceHigh + expr: (neuraldb_database_size_bytes / disk_total_bytes) > 0.80 + for: 5m + labels: + severity: warning + annotations: + summary: "Database storage above 80% capacity" + + - alert: NeuralDBVectorBufferExhausted + expr: neuraldb_hnsw_index_size_bytes > (neuraldb_vector_buffer_size_bytes * 0.90) + for: 5m + labels: + severity: warning + annotations: + summary: "HNSW indexes using >90% of vector_buffer" +``` + +## Built-In Query Statistics + +```sql +-- Top 10 slowest queries +SELECT query, + calls, + round(mean_exec_time::numeric, 2) AS avg_ms, + round(total_exec_time::numeric, 2) AS total_ms, + round(stddev_exec_time::numeric, 2) AS stddev_ms +FROM pg_stat_statements +ORDER BY mean_exec_time DESC +LIMIT 10; + +-- Cache hit ratio (should be >99%) +SELECT + sum(blks_hit) * 100.0 / sum(blks_hit + blks_read) AS cache_hit_ratio +FROM pg_stat_database +WHERE datname != 'template0'; + +-- Lock waits +SELECT pid, query, state, wait_event_type, wait_event, query_start +FROM pg_stat_activity +WHERE wait_event_type = 'Lock' +ORDER BY query_start; +``` + +## Log-Based Alerting + +Forward slow query logs to your SIEM or log aggregation system: + +```ini +# neuraldb.conf +log_destination = 'jsonlog' +log_min_duration_statement = 500 # log queries slower than 500ms +log_line_prefix = '%t [%p] %u@%d ' +``` + +Parse JSON logs in Loki or Elasticsearch and alert when the rate of slow queries exceeds a threshold. diff --git a/sample-sites/neuraldb-docs/pages/ops-scaling.md b/sample-sites/neuraldb-docs/pages/ops-scaling.md new file mode 100644 index 0000000..e55e676 --- /dev/null +++ b/sample-sites/neuraldb-docs/pages/ops-scaling.md @@ -0,0 +1,201 @@ +--- +title: Scaling +sort: 120 +section-id: operations +keywords: scaling, sharding, read replicas, horizontal scaling, capacity planning, performance +description: Scaling NeuralDB horizontally with sharding, read replicas, and capacity planning +language: en +--- + +# Scaling + +NeuralDB is designed to scale horizontally. This page covers adding read replicas for query throughput, sharding for data volume, and capacity planning to avoid resource exhaustion. + +## Vertical Scaling (Scale Up) + +Before adding nodes, ensure you have maximised single-node performance: + +### Memory + +The biggest lever for NeuralDB performance is memory. Ensure: +- `vector_buffer` is large enough to hold all active HNSW graphs +- `shared_buffers` is set to 25% of RAM +- `work_mem` is appropriate for your query patterns + +```sql +-- Check if vectors are being served from disk (slow) vs memory (fast) +SELECT index_name, hnsw_graph_size_bytes, hnsw_in_memory +FROM neuraldb_stat_vector_indexes +ORDER BY hnsw_graph_size_bytes DESC; +``` + +If `hnsw_in_memory = false`, increase `vector_buffer`. + +### CPU + +Vector ANN searches are CPU-bound. Enable parallel query: + +```ini +max_parallel_workers_per_gather = 8 +max_parallel_workers = 16 +``` + +```sql +-- Allow parallel ANN queries for large tables +SET max_parallel_workers_per_gather = 8; +SELECT * FROM large_table ORDER BY embedding <=> $1 LIMIT 10; +``` + +### Storage I/O + +Use NVMe SSDs with high IOPS. Configure the OS: + +```bash +# Increase read-ahead for sequential I/O +sudo blockdev --setra 1024 /dev/nvme0n1 + +# Use deadline/mq-deadline I/O scheduler +echo "mq-deadline" | sudo tee /sys/block/nvme0n1/queue/scheduler + +# Disable transparent huge pages (reduces latency variability) +echo never | sudo tee /sys/kernel/mm/transparent_hugepage/enabled +``` + +## Read Replicas + +Add read replicas to distribute query load. + +### Setting Up Read Replicas + +Follow the [Replication guide](config-replication.md) to add replicas. Each replica can independently serve `SELECT` queries, including vector similarity searches. + +### Client-Side Read Splitting + +Configure your application to route reads to replicas: + +**Python:** +```python +from neuraldb import NeuralDB + +primary = NeuralDB("postgresql://neuraldb:pass@primary:5432/mydb") +replica = NeuralDB("postgresql://neuraldb:pass@replica:5432/mydb") + +def search(query_vector): + # Read goes to replica + return replica.query("SELECT * FROM docs ORDER BY embedding <=> %s LIMIT 10", [query_vector]) + +def insert(content, embedding): + # Write goes to primary + return primary.execute("INSERT INTO docs (content, embedding) VALUES (%s, %s)", [content, embedding]) +``` + +**Connection string with `target_session_attrs`:** +``` +postgresql://neuraldb:pass@primary:5432,replica:5432/mydb?target_session_attrs=prefer-standby +``` + +### Read Replica Scaling Targets + +| Replicas | Approximate peak QPS (1536-dim, 10M vectors) | +|---------|----------------------------------------------| +| 1 primary | 8,000 | +| 1 primary + 2 replicas | 24,000 | +| 1 primary + 4 replicas | 48,000 | +| 1 primary + 8 replicas | 96,000 | + +## Horizontal Sharding + +For datasets exceeding single-node capacity (>50M vectors or >5 TB), shard across multiple primary nodes. + +### Shard Configuration + +```sql +-- Create a sharded cluster (requires NeuralDB Cluster Edition) +SELECT neuraldb_cluster.init_cluster( + shards => 8, + replication_factor => 2 +); + +-- Create a sharded table +CREATE TABLE documents ( + id UUID NOT NULL DEFAULT gen_random_uuid(), + tenant_id UUID NOT NULL, + content TEXT, + embedding VECTOR(1536) +) SHARD BY tenant_id; + +-- Each shard holds ~1/8 of the data +-- All rows with the same tenant_id are colocated on the same shard +``` + +### Cross-Shard Queries + +Cross-shard queries (where the filter doesn't align with the shard key) are automatically parallelised across shards: + +```sql +-- This query executes on all 8 shards in parallel +SELECT id, content, 1 - (embedding <=> $1) AS similarity +FROM documents +ORDER BY embedding <=> $1 +LIMIT 10; +-- Results are merged and re-ranked by the coordinator +``` + +Performance with 8 shards: near-linear scaling. An 8-shard cluster serves ~8ร— the QPS of a single node for cross-shard searches, with ~20% overhead for coordination. + +### Shard Rebalancing + +When adding new shard nodes, rebalance data: + +```sql +-- Rebalance shards (online, non-blocking) +SELECT neuraldb_cluster.rebalance_shards(); + +-- Monitor progress +SELECT * FROM neuraldb_cluster.rebalance_status; +``` + +## Capacity Planning + +### Storage Capacity + +Estimate required storage: + +``` +Row data โ‰ˆ avg_row_size_bytes ร— num_rows ร— 1.3 (index overhead) +Vector data โ‰ˆ dimensions ร— 4 bytes ร— num_vectors +HNSW graph โ‰ˆ dimensions ร— 4 bytes ร— num_vectors ร— 1.3 +WAL โ‰ˆ daily_write_volume ร— wal_retention_days + +Total โ‰ˆ row_data + vector_data + HNSW_graph + WAL + 20% buffer +``` + +Example: 100M rows, 1536 dimensions, 500 bytes average row size: +- Row data: 500B ร— 100M ร— 1.3 โ‰ˆ **65 GB** +- Vector data: 1536 ร— 4B ร— 100M โ‰ˆ **614 GB** +- HNSW graph: 614 GB ร— 1.3 โ‰ˆ **800 GB** (must fit in `vector_buffer`) +- WAL (7 days): 10 GB/day ร— 7 = **70 GB** +- **Total: ~1.6 TB storage, 800 GB RAM for HNSW** + +### Connection Capacity + +``` +max_connections = max_app_connections + pgbouncer_pool_size + replication_slots + 3 (superuser) +``` + +For 500 app connections through PgBouncer with pool size 20: +``` +max_connections = 20 + 10 (replicas) + 3 = 33 +``` + +PgBouncer multiplexes 500 app connections โ†’ 20 database connections. + +### Alert Thresholds + +| Resource | Warning | Critical | +|---------|---------|---------| +| Connections | 80% of max | 95% of max | +| Storage | 70% full | 85% full | +| vector_buffer utilisation | 80% | 90% | +| Replication lag | 30s | 120s | +| Query p99 latency | 500ms | 2000ms | diff --git a/sample-sites/neuraldb-docs/pages/ops-troubleshooting.md b/sample-sites/neuraldb-docs/pages/ops-troubleshooting.md new file mode 100644 index 0000000..313fc93 --- /dev/null +++ b/sample-sites/neuraldb-docs/pages/ops-troubleshooting.md @@ -0,0 +1,226 @@ +--- +title: Troubleshooting +sort: 140 +section-id: operations +keywords: troubleshooting, errors, diagnostics, FAQ, common problems, debug +description: Common NeuralDB errors, diagnostic techniques, and frequently asked questions +language: en +--- + +# Troubleshooting + +This page covers common NeuralDB errors and how to diagnose and resolve them. + +## Connection Issues + +### `FATAL: password authentication failed for user "neuraldb"` + +The password is incorrect or the user doesn't exist. + +```bash +# Reset the password as the OS postgres user +sudo -u neuraldb neuraldb-cli +``` + +```sql +ALTER USER neuraldb PASSWORD 'new-password'; +``` + +Check `pg_hba.conf` โ€” ensure the correct authentication method is used for the client's IP address. + +### `FATAL: no pg_hba.conf entry for host "x.x.x.x", user "neuraldb"` + +The client's IP is not in `pg_hba.conf`: + +``` +# Add to pg_hba.conf +host all all x.x.x.x/32 scram-sha-256 +``` + +Reload: `SELECT pg_reload_conf();` + +### `could not connect to server: Connection refused` + +NeuralDB is not running on the expected host/port: + +```bash +# Check process +systemctl status neuraldb +# or +ps aux | grep neuraldb + +# Check listening port +ss -tlnp | grep 5432 + +# Check logs +journalctl -u neuraldb -n 50 +``` + +### `FATAL: remaining connection slots are reserved for non-replication superuser connections` + +All available connections are consumed. Use PgBouncer to pool connections, or increase `max_connections`: + +```sql +-- Check current connections +SELECT count(*), state, wait_event_type FROM pg_stat_activity GROUP BY state, wait_event_type; + +-- Kill idle connections +SELECT pg_terminate_backend(pid) FROM pg_stat_activity +WHERE state = 'idle' AND state_change < NOW() - INTERVAL '10 minutes'; +``` + +## Vector Query Issues + +### Slow Vector Searches + +If vector queries are slow, check whether the HNSW index is being used: + +```sql +EXPLAIN (ANALYZE, BUFFERS) +SELECT id, embedding <=> '[...]' AS dist +FROM documents +ORDER BY embedding <=> '[...]' +LIMIT 10; +``` + +Look for `Index Scan using documents_embedding_idx` in the plan. If you see `Seq Scan`, the planner may have decided the index is not beneficial. + +Common causes: +1. **Missing LIMIT clause**: The planner only uses the HNSW index for `ORDER BY ... LIMIT` queries. +2. **Too few rows**: For small tables, a sequential scan may be faster. +3. **HNSW graph not in memory**: Check `SELECT * FROM neuraldb_stat_vector_indexes` โ€” if `hnsw_in_memory = false`, increase `vector_buffer`. +4. **ef_search too low**: Increase for better recall at the cost of speed. + +```sql +-- Force index use for debugging +SET enable_seqscan = off; +EXPLAIN ANALYZE SELECT ... ORDER BY embedding <=> $1 LIMIT 10; +SET enable_seqscan = on; +``` + +### `ERROR: expected 1536 dimensions, not 768` + +The vector you are inserting has a different number of dimensions than the column definition: + +```sql +-- Check column definition +\d documents +-- embedding column shows VECTOR(1536) + +-- You are inserting a 768-dimensional vector โ€” check your embedding model +``` + +Ensure your embedding model is consistent. If you need to change models, you must re-embed all existing data. + +### Low Recall on ANN Queries + +If approximate queries are not returning expected results: + +```sql +-- Increase ef_search for higher recall +SET hnsw.ef_search = 200; +SELECT id, content, 1 - (embedding <=> $1) AS similarity +FROM documents +ORDER BY embedding <=> $1 +LIMIT 10; +``` + +Compare against exact search: + +```sql +SET neuraldb.vector_scan = 'exact'; +SELECT id, content FROM documents ORDER BY embedding <=> $1 LIMIT 10; +``` + +If exact search finds results that approximate search misses, increase `ef_search` or rebuild the index with a higher `m` value. + +## Performance Issues + +### High Memory Usage + +```sql +-- Check vector index memory consumption +SELECT index_name, pg_size_pretty(hnsw_graph_size_bytes) AS graph_memory +FROM neuraldb_stat_vector_indexes +ORDER BY hnsw_graph_size_bytes DESC; + +-- Check for shared_buffers usage +SELECT name, setting, unit FROM pg_settings +WHERE name IN ('shared_buffers', 'vector_buffer', 'work_mem'); +``` + +### Disk Space Exhaustion + +```sql +-- Identify large tables and indexes +SELECT tablename, pg_size_pretty(pg_total_relation_size(tablename::regclass)) AS size +FROM pg_tables WHERE schemaname = 'public' +ORDER BY pg_total_relation_size(tablename::regclass) DESC +LIMIT 20; + +-- Check WAL accumulation (often caused by idle replication slots) +SELECT slot_name, active, pg_size_pretty(pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn)) AS lag +FROM pg_replication_slots +ORDER BY pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn) DESC; +``` + +Drop inactive replication slots: + +```sql +SELECT pg_drop_replication_slot('orphaned_slot_name'); +``` + +### Long-Running Queries + +```sql +-- Find queries running longer than 30 seconds +SELECT pid, now() - pg_stat_activity.query_start AS duration, query, state +FROM pg_stat_activity +WHERE state != 'idle' + AND (now() - pg_stat_activity.query_start) > INTERVAL '30 seconds' +ORDER BY duration DESC; + +-- Terminate a specific query +SELECT pg_cancel_backend(pid); -- send SIGINT (graceful) +SELECT pg_terminate_backend(pid); -- send SIGTERM (forceful) +``` + +## Replication Issues + +### Replication Lag Growing + +```sql +-- Check lag on primary +SELECT client_addr, state, sent_lsn - replay_lsn AS lag_bytes +FROM pg_stat_replication; + +-- On replica: check replay lag +SELECT now() - pg_last_xact_replay_timestamp() AS lag; +``` + +Causes: high write load, slow replica hardware, network saturation. Solutions: increase replica hardware, add more replicas, use async replication. + +### `FATAL: the database system is not accepting connections โ€” the database system is starting up` + +NeuralDB is replaying WAL after a crash. Wait for it to complete. Check progress: + +```bash +tail -f /var/log/neuraldb/neuraldb.log +``` + +## FAQ + +**Q: Can I use NeuralDB as a drop-in replacement for PostgreSQL?** +Yes. NeuralDB implements the PostgreSQL wire protocol. Any psql-compatible client, ORM, or tool that works with PostgreSQL will work with NeuralDB. + +**Q: How do I know if my HNSW index is working correctly?** +Run `EXPLAIN ANALYZE` on your vector query and look for `Index Scan using ... hnsw`. Also check `neuraldb_stat_vector_indexes` for `estimated_recall`. + +**Q: What should `vector_buffer` be set to?** +Set it large enough to hold the sum of all HNSW graph sizes. Query `SELECT SUM(hnsw_graph_size_bytes) FROM neuraldb_stat_vector_indexes` to see the current total. + +**Q: Can I store vectors from different embedding models in the same table?** +Only if they have the same dimensionality. Otherwise, use separate columns (e.g., `embedding_openai VECTOR(1536)` and `embedding_cohere VECTOR(1024)`). + +**Q: Is NeuralDB compatible with pgvector extensions?** +NeuralDB includes native support for all pgvector data types (`VECTOR`, `HALFVEC`, `SPARSEVEC`) and operators (`<=>`, `<->`, `<#>`). Applications written for pgvector work without modification. diff --git a/sample-sites/neuraldb-docs/pages/sdk-go.md b/sample-sites/neuraldb-docs/pages/sdk-go.md new file mode 100644 index 0000000..a4b1c55 --- /dev/null +++ b/sample-sites/neuraldb-docs/pages/sdk-go.md @@ -0,0 +1,286 @@ +--- +title: Go SDK +sort: 120 +section-id: client-sdks +keywords: Go, Golang, SDK, client, connection pool, query builder, pgx +description: The NeuralDB Go SDK โ€” installation, connection pooling, and vector query builder +language: en +--- + +# Go SDK + +The NeuralDB Go SDK provides idiomatic Go bindings built on top of `pgx`, the high-performance PostgreSQL driver for Go. + +## Installation + +```bash +go get github.com/neuraldb/neuraldb-go +``` + +Requires Go 1.21+. + +## Connecting + +### Single Connection + +```go +package main + +import ( + "context" + "fmt" + "log" + + "github.com/neuraldb/neuraldb-go" +) + +func main() { + ctx := context.Background() + + client, err := neuraldb.Connect(ctx, "postgresql://neuraldb:password@localhost:5432/mydb") + if err != nil { + log.Fatal("connection failed:", err) + } + defer client.Close(ctx) + + var count int64 + err = client.QueryRow(ctx, "SELECT COUNT(*) FROM documents").Scan(&count) + if err != nil { + log.Fatal(err) + } + fmt.Println("Documents:", count) +} +``` + +### Connection Pool (Recommended) + +```go +import ( + "github.com/neuraldb/neuraldb-go" + "github.com/jackc/pgx/v5/pgxpool" +) + +func NewPool(ctx context.Context) (*neuraldb.Pool, error) { + config, err := pgxpool.ParseConfig(os.Getenv("NEURALDB_URL")) + if err != nil { + return nil, err + } + + config.MaxConns = 20 + config.MinConns = 5 + config.MaxConnIdleTime = 30 * time.Minute + config.MaxConnLifetime = time.Hour + + return neuraldb.NewPool(ctx, config) +} +``` + +## Working with Vectors + +### Defining Vector Types + +```go +package main + +import "github.com/neuraldb/neuraldb-go/types" + +// Create a vector from a float32 slice +v := types.NewVector([]float32{0.023, -0.187, 0.412}) + +// Create from float64 (auto-converted) +v2 := types.NewVectorFromFloat64([]float64{0.023, -0.187, 0.412}) + +// Access the underlying data +floats := v.Slice() // []float32 +dims := v.Dims() // int +``` + +### Inserting a Document with a Vector + +```go +type Document struct { + ID string + Content string + Embedding types.Vector +} + +func InsertDocument(ctx context.Context, pool *neuraldb.Pool, doc Document) error { + _, err := pool.Exec(ctx, + `INSERT INTO documents (id, content, embedding) VALUES ($1, $2, $3)`, + doc.ID, doc.Content, doc.Embedding, + ) + return err +} +``` + +### Vector Similarity Search + +```go +func SemanticSearch(ctx context.Context, pool *neuraldb.Pool, queryEmbedding []float32, limit int) ([]SearchResult, error) { + qv := types.NewVector(queryEmbedding) + + rows, err := pool.Query(ctx, ` + SELECT id, content, 1 - (embedding <=> $1) AS similarity + FROM documents + WHERE embedding IS NOT NULL + ORDER BY embedding <=> $1 + LIMIT $2 + `, qv, limit) + if err != nil { + return nil, fmt.Errorf("query failed: %w", err) + } + defer rows.Close() + + var results []SearchResult + for rows.Next() { + var r SearchResult + err := rows.Scan(&r.ID, &r.Content, &r.Similarity) + if err != nil { + return nil, err + } + results = append(results, r) + } + + return results, rows.Err() +} +``` + +## Query Builder + +The SDK includes an optional query builder for type-safe query construction: + +```go +import "github.com/neuraldb/neuraldb-go/qb" + +// Build a hybrid query +query, args := qb.New(). + Select("id", "name", "price"). + Expr("1 - (embedding <=> $?) AS similarity", queryVector). + From("products"). + Where(qb.Eq("category", "electronics")). + Where(qb.GTE("price", 50)). + Where(qb.LTE("price", 500)). + Where(qb.GT("stock", 0)). + OrderByExpr("embedding <=> $?", queryVector). + Limit(20). + Build() + +rows, err := pool.Query(ctx, query, args...) +``` + +## Batch Operations + +```go +import "github.com/jackc/pgx/v5" + +func BatchInsert(ctx context.Context, pool *neuraldb.Pool, docs []Document) error { + batch := &pgx.Batch{} + + for _, doc := range docs { + batch.Queue( + `INSERT INTO documents (content, embedding) VALUES ($1, $2)`, + doc.Content, doc.Embedding, + ) + } + + results := pool.SendBatch(ctx, batch) + defer results.Close() + + for range docs { + _, err := results.Exec() + if err != nil { + return fmt.Errorf("batch insert error: %w", err) + } + } + + return results.Close() +} +``` + +## Transactions + +```go +func TransactionalInsert(ctx context.Context, pool *neuraldb.Pool, docs []Document) error { + return pool.BeginTxFunc(ctx, pgx.TxOptions{ + IsoLevel: pgx.ReadCommitted, + }, func(tx pgx.Tx) error { + for _, doc := range docs { + _, err := tx.Exec(ctx, + `INSERT INTO documents (content, embedding) VALUES ($1, $2)`, + doc.Content, doc.Embedding, + ) + if err != nil { + return err // auto-rolled back by BeginTxFunc + } + } + + _, err := tx.Exec(ctx, + `UPDATE stats SET doc_count = doc_count + $1`, + len(docs), + ) + return err + }) +} +``` + +## Scanning Results into Structs + +```go +import "github.com/jackc/pgx/v5/pgxscan" + +type Product struct { + ID string `db:"id"` + Name string `db:"name"` + Price float64 `db:"price"` + Similarity float64 `db:"similarity"` +} + +func SearchProducts(ctx context.Context, pool *neuraldb.Pool, qv types.Vector) ([]Product, error) { + var products []Product + err := pgxscan.Select(ctx, pool, &products, ` + SELECT id, name, price, 1 - (embedding <=> $1) AS similarity + FROM products + WHERE available = true + ORDER BY embedding <=> $1 + LIMIT 10 + `, qv) + return products, err +} +``` + +## Context and Cancellation + +All SDK methods accept a `context.Context`. Use it for timeout and cancellation: + +```go +// Set a 5-second query deadline +ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) +defer cancel() + +rows, err := pool.Query(ctx, "SELECT ...", args...) +``` + +## Error Handling + +```go +import ( + "github.com/jackc/pgx/v5/pgconn" + "errors" +) + +_, err := pool.Exec(ctx, "INSERT INTO ...", args...) +if err != nil { + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) { + switch pgErr.Code { + case "23505": // unique_violation + return ErrDuplicate + case "23503": // foreign_key_violation + return ErrForeignKey + default: + return fmt.Errorf("database error %s: %s", pgErr.Code, pgErr.Message) + } + } + return err +} +``` diff --git a/sample-sites/neuraldb-docs/pages/sdk-javascript.md b/sample-sites/neuraldb-docs/pages/sdk-javascript.md new file mode 100644 index 0000000..f65df46 --- /dev/null +++ b/sample-sites/neuraldb-docs/pages/sdk-javascript.md @@ -0,0 +1,277 @@ +--- +title: JavaScript SDK +sort: 110 +section-id: client-sdks +keywords: JavaScript, TypeScript, SDK, Node.js, browser, npm, client +description: The NeuralDB JavaScript/TypeScript SDK for Node.js and browser environments +language: en +--- + +# JavaScript SDK + +The NeuralDB JavaScript SDK provides a fully typed client for Node.js and browser environments. It is built on `pg` (node-postgres) for Node.js and a custom HTTP adapter for edge and browser environments. + +## Installation + +```bash +npm install @neuraldb/client +# or +yarn add @neuraldb/client +# or +pnpm add @neuraldb/client +``` + +## Basic Setup + +### Node.js + +```typescript +import { NeuralDB } from '@neuraldb/client'; + +const client = new NeuralDB({ + connectionString: process.env.NEURALDB_URL!, + // or individual options: + host: 'localhost', + port: 5432, + user: 'neuraldb', + password: 'password', + database: 'mydb', + ssl: { rejectUnauthorized: true }, +}); + +await client.connect(); +``` + +### Edge / Serverless (HTTP mode) + +For Cloudflare Workers, Vercel Edge, and browser environments, use the HTTP adapter: + +```typescript +import { NeuralDB, HttpAdapter } from '@neuraldb/client'; + +const client = new NeuralDB({ + adapter: new HttpAdapter({ + url: process.env.NEURALDB_HTTP_URL!, + apiKey: process.env.NEURALDB_API_KEY!, + }), +}); +``` + +### Connection Pool + +```typescript +import { NeuralDBPool } from '@neuraldb/client'; + +const pool = new NeuralDBPool({ + connectionString: process.env.NEURALDB_URL!, + max: 20, + idleTimeoutMillis: 30000, + connectionTimeoutMillis: 5000, +}); +``` + +## Executing Queries + +```typescript +// Simple query โ€” returns QueryResult +const result = await client.query('SELECT id, content FROM documents LIMIT 10'); +const rows = result.rows; // typed as any[] + +// Parameterised query +const { rows } = await client.query<{ id: string; content: string }>( + 'SELECT id, content FROM documents WHERE source = $1 LIMIT $2', + ['web-scraper', 20] +); +``` + +## Vector Operations + +### Inserting Vectors + +```typescript +import { toVector } from '@neuraldb/client'; + +const embedding = [0.023, -0.187, 0.412, /* 1536 values */]; + +await client.query( + 'INSERT INTO documents (content, embedding) VALUES ($1, $2)', + ['My document content', toVector(embedding)] +); +``` + +### Similarity Search + +```typescript +import OpenAI from 'openai'; +import { toVector } from '@neuraldb/client'; + +const openai = new OpenAI(); + +async function semanticSearch(query: string, limit = 10) { + const embeddingResponse = await openai.embeddings.create({ + model: 'text-embedding-3-small', + input: query, + }); + + const queryVector = embeddingResponse.data[0].embedding; + + const { rows } = await client.query<{ + id: string; + content: string; + similarity: number; + }>( + `SELECT id, content, 1 - (embedding <=> $1) AS similarity + FROM documents + WHERE embedding IS NOT NULL + ORDER BY embedding <=> $1 + LIMIT $2`, + [toVector(queryVector), limit] + ); + + return rows; +} +``` + +### Hybrid Search + +```typescript +async function hybridSearch(query: string, filters: Record, limit = 10) { + const queryVector = await generateEmbedding(query); + + const conditions: string[] = []; + const params: unknown[] = [toVector(queryVector)]; + + Object.entries(filters).forEach(([key, value]) => { + params.push(value); + conditions.push(`${key} = $${params.length}`); + }); + + const whereClause = conditions.length > 0 + ? 'WHERE ' + conditions.join(' AND ') + : ''; + + params.push(limit); + + const { rows } = await client.query( + `SELECT id, content, 1 - (embedding <=> $1) AS similarity + FROM documents + ${whereClause} + ORDER BY embedding <=> $1 + LIMIT $${params.length}`, + params + ); + + return rows; +} +``` + +## High-Level Document API + +The SDK includes a higher-level document management API: + +```typescript +import { DocumentStore } from '@neuraldb/client'; + +const store = new DocumentStore(client, { + table: 'documents', + embeddingColumn: 'embedding', + contentColumn: 'content', + embeddingModel: { + provider: 'openai', + model: 'text-embedding-3-small', + apiKey: process.env.OPENAI_API_KEY!, + }, +}); + +// Add documents (auto-generates embeddings) +await store.add([ + { content: 'First document', metadata: { source: 'web' } }, + { content: 'Second document', metadata: { source: 'pdf' } }, +]); + +// Search +const results = await store.search('query text', { + limit: 10, + filter: { source: 'web' }, + minSimilarity: 0.7, +}); + +// Delete +await store.delete({ filter: { source: 'web' } }); +``` + +## Transactions + +```typescript +const pgClient = await pool.connect(); + +try { + await pgClient.query('BEGIN'); + + await pgClient.query( + 'INSERT INTO documents (content, embedding) VALUES ($1, $2)', + ['Content', toVector(embedding)] + ); + + await pgClient.query( + 'UPDATE stats SET doc_count = doc_count + 1 WHERE id = $1', + [statsId] + ); + + await pgClient.query('COMMIT'); +} catch (error) { + await pgClient.query('ROLLBACK'); + throw error; +} finally { + pgClient.release(); +} +``` + +## Streaming Results + +For large result sets, stream rows to avoid loading all data into memory: + +```typescript +const stream = client.queryStream( + 'SELECT id, content, embedding FROM documents WHERE source = $1', + ['web-scraper'] +); + +for await (const row of stream) { + await processDocument(row); +} +``` + +## Type Definitions + +```typescript +import type { Vector, QueryResult, PoolClient } from '@neuraldb/client'; + +interface Document { + id: string; + content: string; + embedding: Vector | null; + created_at: Date; +} + +const { rows } = await client.query( + 'SELECT * FROM documents WHERE id = $1', + [id] +); +``` + +## Error Handling + +```typescript +import { NeuralDBError, ConnectionError, QueryError } from '@neuraldb/client'; + +try { + await client.query('SELECT * FROM nonexistent_table'); +} catch (error) { + if (error instanceof QueryError) { + console.error('SQL error:', error.message, 'Code:', error.code); + } else if (error instanceof ConnectionError) { + console.error('Connection failed:', error.message); + } +} +``` diff --git a/sample-sites/neuraldb-docs/pages/sdk-python.md b/sample-sites/neuraldb-docs/pages/sdk-python.md new file mode 100644 index 0000000..f2b6b36 --- /dev/null +++ b/sample-sites/neuraldb-docs/pages/sdk-python.md @@ -0,0 +1,277 @@ +--- +title: Python SDK +sort: 100 +section-id: client-sdks +keywords: Python, SDK, client, connection, CRUD, vector operations, psycopg +description: Installing and using the NeuralDB Python SDK โ€” connection, CRUD, and vector operations +language: en +--- + +# Python SDK + +The NeuralDB Python SDK provides a high-level client for Python applications. It is built on top of `psycopg3` (the PostgreSQL adapter) with NeuralDB-specific helpers for vector operations, embedding generation, and batch ingestion. + +## Installation + +```bash +pip install neuraldb +# or +pip install neuraldb[asyncio] # includes async support +pip install neuraldb[all] # includes all optional extras +``` + +### Requirements + +- Python 3.10+ +- libpq (PostgreSQL client library) + +On Ubuntu: `sudo apt install libpq-dev` +On macOS: `brew install libpq` + +## Connecting + +### Synchronous Client + +```python +from neuraldb import NeuralDB + +# From connection string +client = NeuralDB("postgresql://neuraldb:password@localhost:5432/mydb") + +# From parameters +client = NeuralDB( + host="localhost", + port=5432, + user="neuraldb", + password="password", + database="mydb", + sslmode="require", +) + +# Context manager (auto-closes connection) +with NeuralDB("postgresql://...") as client: + result = client.query("SELECT 1") +``` + +### Async Client + +```python +import asyncio +from neuraldb import AsyncNeuralDB + +async def main(): + async with AsyncNeuralDB("postgresql://neuraldb:password@localhost/mydb") as client: + result = await client.query("SELECT 1") + print(result) + +asyncio.run(main()) +``` + +### Connection Pool + +```python +from neuraldb import NeuralDBPool + +pool = NeuralDBPool( + "postgresql://neuraldb:password@localhost/mydb", + min_size=5, + max_size=20, +) + +with pool.acquire() as client: + result = client.query("SELECT COUNT(*) FROM documents") +``` + +## Schema Operations + +```python +# Create a table with a vector column +client.execute(""" + CREATE TABLE IF NOT EXISTS documents ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + content TEXT NOT NULL, + source TEXT, + embedding VECTOR(1536), + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() + ) +""") + +# Create a vector index +client.execute(""" + CREATE INDEX IF NOT EXISTS documents_embedding_idx + ON documents USING hnsw (embedding vector_cosine_ops) + WITH (m = 16, ef_construction = 64) +""") +``` + +## CRUD Operations + +### Insert + +```python +from neuraldb import Vector + +# Insert with pre-computed embedding +client.execute( + "INSERT INTO documents (content, source, embedding) VALUES (%s, %s, %s)", + ("My document content", "web-scraper", Vector([0.023, -0.187, 0.412, ...])) +) + +# Insert many (batched for efficiency) +docs = [ + ("Content A", "source-1", Vector([...])), + ("Content B", "source-2", Vector([...])), + ("Content C", "source-1", Vector([...])), +] +client.executemany( + "INSERT INTO documents (content, source, embedding) VALUES (%s, %s, %s)", + docs +) +``` + +### Query + +```python +# Standard query โ€” returns list of Row objects +rows = client.query("SELECT id, content FROM documents WHERE source = %s", ("web-scraper",)) + +for row in rows: + print(row["id"], row["content"]) + +# As dicts +rows = client.query( + "SELECT * FROM documents LIMIT 10", + row_factory="dict" +) + +# As named tuples +rows = client.query( + "SELECT id, content FROM documents LIMIT 10", + row_factory="namedtuple" +) +``` + +### Vector Search + +```python +import openai + +# Generate query embedding +query_text = "high-performance wireless headphones" +query_embedding = openai.embeddings.create( + model="text-embedding-3-small", + input=query_text +).data[0].embedding + +# Semantic search +results = client.query(""" + SELECT id, content, 1 - (embedding <=> %s) AS similarity + FROM documents + WHERE embedding IS NOT NULL + ORDER BY embedding <=> %s + LIMIT 10 +""", (Vector(query_embedding), Vector(query_embedding))) + +for row in results: + print(f"{row['similarity']:.3f}: {row['content'][:100]}") +``` + +### Using the High-Level Search API + +```python +from neuraldb import VectorSearch + +searcher = VectorSearch(client, table="documents", embedding_column="embedding") + +results = searcher.search( + query_vector=query_embedding, + limit=10, + filters={"source": "web-scraper"}, + metric="cosine", +) +``` + +### Update + +```python +client.execute( + "UPDATE documents SET content = %s, embedding = %s WHERE id = %s", + ("Updated content", Vector(new_embedding), doc_id) +) +``` + +### Delete + +```python +client.execute("DELETE FROM documents WHERE id = %s", (doc_id,)) +``` + +## Transactions + +```python +with client.transaction(): + client.execute("INSERT INTO documents (content, embedding) VALUES (%s, %s)", (content, Vector(embedding))) + client.execute("UPDATE stats SET count = count + 1") + # Auto-commits on exit, rolls back on exception +``` + +Explicit control: + +```python +with client.transaction() as txn: + try: + client.execute("INSERT ...") + client.execute("UPDATE ...") + txn.commit() + except Exception: + txn.rollback() + raise +``` + +## Bulk Ingestion + +For high-throughput ingestion, use the `BulkIngestor`: + +```python +from neuraldb import BulkIngestor + +ingestor = BulkIngestor( + client, + table="documents", + columns=["content", "source", "embedding"], + batch_size=1000, # insert in batches of 1000 + embedding_model="openai/text-embedding-3-small", # auto-generate embeddings + embedding_column="embedding", + text_column="content", +) + +docs = [ + {"content": "Document text here", "source": "source-1"}, + {"content": "Another document", "source": "source-2"}, + # ... thousands more +] + +with ingestor as ing: + for doc in docs: + ing.add(doc) + # Flushes remaining rows and commits on context exit + +print(f"Ingested {ingestor.total_inserted} documents") +``` + +## Type Handling + +The SDK provides type adapters for NeuralDB types: + +```python +from neuraldb.types import Vector, HalfVector, SparseVector + +# Dense vector +v = Vector([0.1, 0.2, 0.3]) + +# Half-precision vector (less memory) +hv = HalfVector([0.1, 0.2, 0.3]) + +# Sparse vector +sv = SparseVector({0: 0.5, 15: 0.3, 200: 0.8}, dimensions=384) +``` diff --git a/sample-sites/neuraldb-docs/pages/sdk-rest.md b/sample-sites/neuraldb-docs/pages/sdk-rest.md new file mode 100644 index 0000000..005a52c --- /dev/null +++ b/sample-sites/neuraldb-docs/pages/sdk-rest.md @@ -0,0 +1,283 @@ +--- +title: REST API +sort: 130 +section-id: client-sdks +keywords: REST API, HTTP, endpoints, authentication, JSON, API +description: NeuralDB REST API reference โ€” all endpoints, authentication headers, and response formats +language: en +--- + +# REST API + +NeuralDB provides an HTTP REST API for environments where a direct database connection is not practical (browser clients, third-party integrations, webhooks). The REST API is a thin HTTP wrapper over NQL. + +## Base URL + +``` +https://your-neuraldb-host:8080/api/v1 +``` + +For NeuralDB Cloud: + +``` +https://[cluster-id].cloud.neuraldb.io/api/v1 +``` + +## Authentication + +All REST API requests require an API key in the `Authorization` header: + +``` +Authorization: Bearer ndb_live_your_api_key_here +``` + +Or as a query parameter (less secure, avoid in production): + +``` +?api_key=ndb_live_your_api_key_here +``` + +Create API keys in the NeuralDB CLI or the Cloud dashboard: + +```bash +neuraldb-cli -c "SELECT neuraldb_apikeys.create_key(label => 'my-app', role => 'app_readwrite')" +``` + +## Query Endpoint + +Execute any NQL query: + +### `POST /query` + +```http +POST /api/v1/query +Content-Type: application/json +Authorization: Bearer ndb_live_... + +{ + "query": "SELECT id, content, 1 - (embedding <=> $1) AS similarity FROM documents ORDER BY embedding <=> $1 LIMIT 5", + "params": [[0.023, -0.187, 0.412]], + "database": "mydb" +} +``` + +**Response:** + +```json +{ + "rows": [ + { "id": "uuid-1", "content": "First document", "similarity": 0.923 }, + { "id": "uuid-2", "content": "Second document", "similarity": 0.891 } + ], + "rowCount": 2, + "fields": [ + { "name": "id", "dataTypeID": 2950 }, + { "name": "content", "dataTypeID": 25 }, + { "name": "similarity", "dataTypeID": 701 } + ], + "executionTimeMs": 3.2 +} +``` + +### Query Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `query` | string | Yes | NQL query string | +| `params` | array | No | Query parameters (replaces $1, $2, ...) | +| `database` | string | No | Target database (default: `neuraldb`) | +| `timeout_ms` | integer | No | Query timeout in milliseconds | +| `explain` | boolean | No | Return query execution plan | + +## Document Endpoints + +Higher-level CRUD endpoints for document management: + +### `POST /collections/:collection/documents` + +Insert documents (auto-generates embeddings if configured): + +```http +POST /api/v1/collections/my_docs/documents +Content-Type: application/json +Authorization: Bearer ndb_live_... + +{ + "documents": [ + { + "content": "NeuralDB is an AI-native database", + "metadata": { "source": "blog", "category": "technology" } + } + ], + "embedding_model": "openai/text-embedding-3-small" +} +``` + +**Response:** + +```json +{ + "inserted": 1, + "ids": ["550e8400-e29b-41d4-a716-446655440000"] +} +``` + +### `POST /collections/:collection/search` + +Vector similarity search: + +```http +POST /api/v1/collections/my_docs/search +Content-Type: application/json +Authorization: Bearer ndb_live_... + +{ + "query": "AI-native database for semantic search", + "limit": 10, + "min_similarity": 0.7, + "filters": { + "category": "technology" + }, + "embedding_model": "openai/text-embedding-3-small", + "include_metadata": true +} +``` + +**Response:** + +```json +{ + "results": [ + { + "id": "550e8400-e29b-41d4-a716-446655440000", + "content": "NeuralDB is an AI-native database", + "similarity": 0.956, + "metadata": { "source": "blog", "category": "technology" } + } + ], + "query_embedding_model": "openai/text-embedding-3-small", + "latency_ms": 2.8 +} +``` + +### `GET /collections/:collection/documents/:id` + +Retrieve a document by ID: + +```http +GET /api/v1/collections/my_docs/documents/550e8400-e29b-41d4-a716-446655440000 +Authorization: Bearer ndb_live_... +``` + +### `PUT /collections/:collection/documents/:id` + +Update a document: + +```http +PUT /api/v1/collections/my_docs/documents/550e8400-... +Content-Type: application/json +Authorization: Bearer ndb_live_... + +{ + "content": "Updated content", + "metadata": { "source": "blog", "updated": true }, + "regenerate_embedding": true +} +``` + +### `DELETE /collections/:collection/documents/:id` + +Delete a document: + +```http +DELETE /api/v1/collections/my_docs/documents/550e8400-... +Authorization: Bearer ndb_live_... +``` + +## Batch Operations + +### `POST /collections/:collection/documents/batch` + +Insert up to 1,000 documents in a single request: + +```http +POST /api/v1/collections/my_docs/documents/batch +Content-Type: application/json + +{ + "documents": [ + { "content": "Document 1", "metadata": {} }, + { "content": "Document 2", "metadata": {} } + ] +} +``` + +## Error Responses + +All errors return a consistent JSON structure: + +```json +{ + "error": { + "code": "QUERY_ERROR", + "message": "relation \"nonexistent\" does not exist", + "detail": "ERROR: relation \"nonexistent\" does not exist at character 15", + "status": 400 + } +} +``` + +### Error Codes + +| HTTP Status | Error Code | Description | +|-------------|-----------|-------------| +| 400 | `QUERY_ERROR` | Invalid NQL query | +| 400 | `VALIDATION_ERROR` | Request body validation failed | +| 401 | `UNAUTHORIZED` | Missing or invalid API key | +| 403 | `FORBIDDEN` | Insufficient role permissions | +| 404 | `NOT_FOUND` | Document or collection not found | +| 429 | `RATE_LIMITED` | Too many requests | +| 500 | `INTERNAL_ERROR` | Server error | +| 503 | `DATABASE_UNAVAILABLE` | NeuralDB is unreachable | + +## Rate Limits + +| Plan | Queries/min | Documents/min | +|------|------------|--------------| +| Starter | 30 | 100 | +| Developer | 300 | 1,000 | +| Business | 3,000 | 10,000 | +| Business Plus | 10,000 | 50,000 | + +Rate limit headers are included in all responses: + +``` +X-RateLimit-Limit: 3000 +X-RateLimit-Remaining: 2847 +X-RateLimit-Reset: 1716998460 +``` + +## Pagination + +Use `limit` and `offset` (or cursor-based pagination for large datasets): + +```json +{ + "query": "SELECT * FROM documents WHERE source = $1 ORDER BY created_at DESC", + "params": ["web-scraper"], + "limit": 20, + "offset": 40 +} +``` + +Response includes pagination metadata: + +```json +{ + "rows": [...], + "rowCount": 20, + "totalCount": 1500, + "hasMore": true, + "nextCursor": "eyJpZCI6Ii4uLiIsImNyZWF0ZWRfYXQiOiIuLi4ifQ==" +} +``` diff --git a/sample-sites/neuraldb-docs/search.json b/sample-sites/neuraldb-docs/search.json new file mode 100644 index 0000000..f80486c --- /dev/null +++ b/sample-sites/neuraldb-docs/search.json @@ -0,0 +1,314 @@ +[ + { + "file": "pages/architecture.md", + "title": "Architecture", + "section-id": "overview", + "keywords": "architecture, storage engine, query planner, replication, WAL, HNSW", + "description": "NeuralDB internal architecture \u2014 storage engine, query planner, and replication", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Architecture\n\n![NeuralDB Architecture](assets/images/architecture.jpg)\n\nNeuralDB is built on a custom storage engine that co-locates relational and vector data, with a query planner that understands both SQL predicates and vector similarity operations natively.\n\n## High-Level Architecture\n\n```\nClient (psql / SDK / REST)\n \u2502\n \u25bc\n\u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510\n\u2502 Connection Layer \u2502\n\u2502 (PostgreSQL Wire Protocol compatible) \u2502\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n \u2502\n \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u25bc\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510\n \u2502 Query Parser \u2502\n \u2502 (SQL + NQL ext.) \u2502\n \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n \u2502\n \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u25bc\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510\n \u2502 Semantic Planner \u2502\u25c4\u2500\u2500 Statistics + Index Metadata\n \u2502 (hybrid cost model) \u2502\n \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n \u2502\n \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u25bc\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510\n \u2502 Execution Engine \u2502\n \u2502 \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510 \u2502\n \u2502 \u2502 SQL Executor \u2502 \u2502\n \u2502 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518 \u2502\n \u2502 \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510 \u2502\n \u2502 \u2502 ANN Executor \u2502 \u2502\n \u2502 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518 \u2502\n \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n \u2502\n \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u25bc\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510\n \u2502 Storage Engine \u2502\n \u2502 \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510 \u2502\n \u2502 \u2502 Row Store \u2502 \u2502\u25c4\u2500\u2500 SST Files (columnar)\n \u2502 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518 \u2502\n \u2502 \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510 \u2502\n \u2502 \u2502 Vector Store \u2502 \u2502\u25c4\u2500\u2500 HNSW Graph Files\n \u2502 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518 \u2502\n \u2502 \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510 \u2502\n \u2502 \u2502 WAL \u2502 \u2502\u25c4\u2500\u2500 Write-Ahead Log\n \u2502 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518 \u2502\n \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n```\n\n## Storage Engine\n\n### Row Store\n\nNeuralDB's row store uses a Log-Structured Merge-tree (LSM) architecture inspired by RocksDB. Data is written to an in-memory write buffer (MemTable), which is periodically flushed to sorted string tables (SSTables) on disk. Background compaction merges SSTables and reclaims space.\n\nKey properties:\n- **Write-optimised**: writes are sequential, not random \u2014 excellent NVMe utilisation\n- **Columnar format**: SSTables store data column-by-column for fast analytical scans\n- **Compression**: LZ4 by default, Zstd for archival storage \u2014 typically 3\u20136\u00d7 compression ratio\n\n### Vector Store\n\nVectors are stored separately from rows in a Vector Store. The Vector Store maintains:\n\n1. **Raw vector data** \u2014 the float32 arrays, stored in compressed pages\n2. **HNSW graph** \u2014 the in-memory navigation graph for ANN search\n\nThe HNSW graph is loaded into memory on startup and kept warm. Memory required \u2248 `num_vectors \u00d7 dimensions \u00d7 4 bytes \u00d7 1.3` (1.3\u00d7 overhead for the graph structure).\n\nFor a 10M-row table with 1536-dimensional embeddings: `10M \u00d7 1536 \u00d7 4 \u00d7 1.3 \u2248 80 GB`. Plan memory accordingly.\n\n### Write-Ahead Log (WAL)\n\nAll writes (row and vector) are first written to the WAL before being applied to the storage engine. The WAL provides:\n\n- **Durability**: committed transactions survive crashes\n- **Replication**: replicas apply the WAL stream from the primary\n- **Point-in-time recovery (PITR)**: archive the WAL to recover to any point in time\n\nWAL segments are 128 MB by default and are archived to the configured storage backend (local disk, S3, GCS) upon rotation.\n\n## Query Planner\n\nThe Semantic Planner extends a PostgreSQL-compatible query planner with understanding of vector operations.\n\n### Hybrid Cost Model\n\nFor hybrid queries (vector + relational), the planner considers two physical plans:\n\n**Plan A: Pre-filter**\n```\nFilter(price < 100) \u2192 ANN(embedding, k=10)\n```\nCost: selectivity \u00d7 full_scan_cost + ANN_cost(filtered_set)\n\n**Plan B: Post-filter**\n```\nANN(embedding, k=10\u00d7estimated_filter_ratio) \u2192 Filter(price < 100)\n```\nCost: ANN_cost(full_index) + filter_cost\n\nThe planner uses column statistics (histogram, null fraction, distinct values) and vector index parameters to estimate costs. It picks the plan with the lower estimated cost.\n\n### Index Types\n\nNeuralDB supports the following index types:\n\n| Index | Data | Purpose |\n|-------|------|---------|\n| B-tree | Scalar columns | Equality, range queries |\n| Hash | Scalar columns | Equality only (faster than B-tree) |\n| GIN | JSON, arrays | Containment queries |\n| HNSW | VECTOR columns | Approximate nearest neighbour |\n| IVF-Flat | VECTOR columns | High-recall exact-ish search |\n| BRIN | Timestamp columns | Range scans on append-only data |\n\n## Replication\n\nNeuralDB uses streaming replication. The primary continuously ships WAL segments to replicas, which apply them in order.\n\n### Synchronous vs Asynchronous Replication\n\n```sql\n-- Set replication mode per-transaction\nSET synchronous_commit = 'on'; -- wait for WAL to reach all sync replicas (safest)\nSET synchronous_commit = 'local'; -- wait for local WAL flush only (faster)\nSET synchronous_commit = 'off'; -- don't wait (fastest, small durability window)\n```\n\n### Read Replicas\n\nReplicas accept `SELECT` queries. Direct read-heavy workloads to replicas:\n\n```\np" + }, + { + "file": "pages/comparison.md", + "title": "Comparison", + "section-id": "overview", + "keywords": "comparison, Postgres, pgvector, Pinecone, Weaviate, MongoDB, alternatives", + "description": "How NeuralDB compares to Postgres+pgvector, Pinecone, Weaviate, and MongoDB Atlas Vector Search", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Comparison\n\nThis page provides an honest comparison of NeuralDB against the most common alternatives for applications that need vector search.\n\n## NeuralDB vs PostgreSQL + pgvector\n\npgvector is the most popular way to add vector search to an existing PostgreSQL deployment. If you already run Postgres, the low barrier to entry is attractive. Here is where the two diverge:\n\n| Feature | NeuralDB | Postgres + pgvector |\n|---------|---------|---------------------|\n| Vector index algorithm | HNSW (native) | HNSW, IVFFlat |\n| Max dimensions | 65,536 | 16,000 |\n| Index build speed | Native Rust (fast) | C extension (moderate) |\n| Parallel index builds | Yes | Limited |\n| Vector data memory isolation | Dedicated vector buffer pool | Shared with row pages |\n| Horizontal sharding | Built-in | Manual (Citus, Patroni) |\n| Automatic embeddings | Yes | No |\n| Multi-modal vectors | Yes | No (one VECTOR column type) |\n| Streaming ingestion | Yes | No |\n| Read replica for vector queries | Automatic routing | Manual |\n\n**When to choose Postgres + pgvector:** You already have a Postgres deployment, your dataset is under 10M vectors, and you do not need automatic embeddings or horizontal scaling. The operational overhead of a new database system is not worth it.\n\n**When to choose NeuralDB:** Your vector dataset exceeds 10M rows, you need horizontal sharding, you want automatic embedding pipelines, or you are starting a new project and want a purpose-built system.\n\n## NeuralDB vs Pinecone\n\nPinecone is a fully managed, purpose-built vector database. It excels at pure vector search at massive scale.\n\n| Feature | NeuralDB | Pinecone |\n|---------|---------|---------|\n| Relational data | Full SQL | Metadata filters only |\n| Hybrid queries | Single query, query planner | Metadata post-filter |\n| ACID transactions | Yes | No |\n| SQL interface | Yes | Proprietary API |\n| Self-hosted option | Yes | No |\n| Pricing model | Infrastructure cost | Per-request + storage |\n| Latency (p99, 1M vectors) | ~5ms | ~10ms (managed) |\n| Data gravity | Stays in your infra | Vendor-managed |\n\n**When to choose Pinecone:** You need a fully managed solution with no operational overhead, your workload is pure vector search with simple metadata filtering, and you are comfortable with a vendor-specific API and pricing model.\n\n**When to choose NeuralDB:** You need relational data co-located with vectors, ACID transactions, SQL compatibility, self-hosting, or lower total cost of ownership at scale.\n\n## NeuralDB vs Weaviate\n\nWeaviate is an open-source vector database with a GraphQL-based query language and built-in module support for embedding generation.\n\n| Feature | NeuralDB | Weaviate |\n|---------|---------|---------|\n| Query language | SQL (NQL) | GraphQL |\n| Relational joins | Yes | No |\n| ACID transactions | Yes | Eventually consistent |\n| SQL wire compatibility | PostgreSQL wire protocol | Proprietary |\n| Embedding modules | Yes | Yes (vectorizers) |\n| BM25 hybrid search | Yes | Yes |\n| Multi-tenancy | Row-level, schema-level | Class-level |\n| Replication | Sync + async | Eventual |\n\n**When to choose Weaviate:** You want an open-source solution with a rich ecosystem of vectorizer modules and a GraphQL interface. If your team is more comfortable with graph-shaped queries than SQL, Weaviate is a natural fit.\n\n**When to choose NeuralDB:** You need SQL, transactional guarantees, relational joins between your vector data and other structured data, or PostgreSQL wire protocol compatibility (so existing tools like dbt, Metabase, and psql work out of the box).\n\n## NeuralDB vs MongoDB Atlas Vector Search\n\nMongoDB Atlas added vector search as an extension to its document model. It is a convenient choice if you already run Atlas.\n\n| Feature | NeuralDB | MongoDB Atlas Vector Search |\n|---------|---------|---------------------------|\n| Data model | Relational + vector | Document + vector |\n| Query language | SQL | MQL (MongoDB Query Language) |\n| ACID transactions | Yes (all operations) | Yes (within a session) |\n| Horizontal scaling | Native sharding | Atlas sharding |\n| Vector index type | HNSW | ENN (exact), HNSW |\n| Full-text + vector hybrid | Yes | Yes (Atlas Search) |\n| Self-hosted | Yes | Atlas only |\n\n**When to choose MongoDB Atlas Vector Search:** Your application already uses MongoDB and you want to add vector search without changing your data model or infrastructure. The document model maps well to semi-structured data.\n\n**When to choose NeuralDB:** You need relational data integrity, SQL, lower query latency, or the ability to self-host. If your data is inherently tabular (rather than document-shaped), NeuralDB's relational model will be a better fit.\n\n## Performance Benchmarks\n\nThe following benchmarks were run against 10M 1536-dimensional vectors on equivalent hardware (32 vCPU, 128 GB RAM, NVMe SSD):\n\n| System | QPS (recall@95%) | p50 latency | p99 latency | Index build time |\n|--------|-----------------|-------------|-------------|-----------------|\n|" + }, + { + "file": "pages/concepts.md", + "title": "Core Concepts", + "section-id": "overview", + "keywords": "concepts, vectors, embeddings, hybrid queries, nodes, HNSW, ANN", + "description": "Core NeuralDB concepts \u2014 vectors, embeddings, hybrid queries, and the node model", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Core Concepts\n\nUnderstanding these fundamental concepts will help you use NeuralDB effectively and make good architectural decisions for your application.\n\n## Vectors and Embeddings\n\nA **vector** is an ordered list of floating-point numbers \u2014 a point in high-dimensional space. In NeuralDB, vectors are used to represent the semantic meaning of data.\n\nAn **embedding** is a vector produced by a machine learning model that encodes the semantic meaning of its input. Similar inputs produce vectors that are close together in the embedding space. For example, the sentences \"I love dogs\" and \"I adore canines\" will produce embeddings that are close to each other, even though they share no words.\n\nNeuralDB stores embeddings as `VECTOR(n)` columns, where `n` is the dimensionality (the number of float32 values). Common dimensionalities:\n\n| Model | Dimensions |\n|-------|-----------|\n| OpenAI text-embedding-3-small | 1536 |\n| OpenAI text-embedding-3-large | 3072 |\n| Cohere embed-english-v3.0 | 1024 |\n| Google text-embedding-004 | 768 |\n| BAAI/bge-m3 | 1024 |\n\n## Distance Metrics\n\nNeuralDB computes similarity between two vectors using one of three distance metrics:\n\n### Cosine Similarity\n\nMeasures the angle between two vectors. Ranges from -1 (opposite) to 1 (identical). Ideal for text embeddings produced by models trained with cosine objectives:\n\n```\ncosine_similarity(a, b) = (a \u00b7 b) / (|a| \u00d7 |b|)\n```\n\nIn NQL, use the `<=>` operator or `COSINE_SIMILARITY()` function.\n\n### Dot Product\n\nMeasures the product of vector magnitudes and the angle between them. Used when vectors are not normalised and magnitude carries information (e.g., collaborative filtering):\n\n```\ndot_product(a, b) = \u03a3(a\u1d62 \u00d7 b\u1d62)\n```\n\nIn NQL, use the `<#>` operator or `DOT_PRODUCT()` function.\n\n### Euclidean Distance (L2)\n\nMeasures straight-line distance between two points. Lower is more similar. Useful for spatial data and image embeddings:\n\n```\nl2_distance(a, b) = \u221a(\u03a3(a\u1d62 - b\u1d62)\u00b2)\n```\n\nIn NQL, use the `<->` operator or `L2_DISTANCE()` function.\n\n## Vector Indexes\n\nNeuralDB builds vector indexes using the **HNSW (Hierarchical Navigable Small World)** algorithm. HNSW provides:\n\n- Sub-linear approximate nearest neighbour (ANN) search\n- Configurable trade-off between speed and recall\n- Incremental updates (no full rebuild needed when inserting)\n\n### HNSW Parameters\n\nWhen creating a vector index:\n\n```sql\nCREATE INDEX ON documents\nUSING hnsw (embedding vector_cosine_ops)\nWITH (m = 16, ef_construction = 64);\n```\n\n| Parameter | Description | Default |\n|-----------|-------------|---------|\n| `m` | Number of bi-directional links per node. Higher = better recall, more memory | 16 |\n| `ef_construction` | Size of candidate set during index construction. Higher = better quality, slower build | 64 |\n| `ef_search` | Size of candidate set at query time. Set per-query with `SET hnsw.ef_search = 100` | 40 |\n\n### Exact vs Approximate Search\n\nBy default, vector queries use the HNSW index (approximate). For exact results (slower but 100% recall), use:\n\n```sql\nSET neuraldb.vector_scan = 'exact';\nSELECT * FROM documents ORDER BY embedding <=> :query LIMIT 10;\n```\n\n## Hybrid Queries\n\nA hybrid query combines vector similarity with relational predicates in a single query plan. The NeuralDB query planner evaluates two strategies and picks the cheaper one:\n\n1. **Pre-filter then search** \u2014 apply relational filters first to reduce the candidate set, then run ANN search on the filtered set\n2. **Post-filter** \u2014 run ANN search to get top-k candidates, then apply relational filters\n\nNeuralDB automatically selects the optimal strategy based on selectivity estimates. You can hint the planner:\n\n```sql\nSELECT * FROM documents\nWHERE /*+ PREFILTER */ category = 'news'\nORDER BY embedding <=> :query\nLIMIT 10;\n```\n\n## Tables and Schemas\n\nNeuralDB is schema-based, like PostgreSQL. Everything lives inside a database \u2192 schema \u2192 table hierarchy:\n\n```sql\nCREATE DATABASE my_app;\n\\c my_app\n\nCREATE SCHEMA vectors;\nCREATE SCHEMA metadata;\n\nCREATE TABLE vectors.documents (\n id UUID PRIMARY KEY DEFAULT gen_random_uuid(),\n content TEXT NOT NULL,\n embedding VECTOR(1536),\n schema_id TEXT REFERENCES metadata.schemas(id),\n created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()\n);\n```\n\n## Nodes\n\nNeuralDB is a distributed system. A **node** is a single NeuralDB process. Nodes have one of three roles:\n\n| Role | Responsibilities |\n|------|----------------|\n| **Primary** | Accepts reads and writes, coordinates transactions |\n| **Replica** | Accepts reads, replicates writes from primary |\n| **Index** | Maintains vector indexes for shard(s), offloads ANN queries |\n\nIn a single-node deployment, one process takes all three roles.\n\n## Sharding\n\nNeuralDB shards data by `shard_key`. By default, the primary key is used as the shard key:\n\n```sql\nCREATE TABLE events (\n id UUID PRIMARY KEY,\n tenant_id UUID NOT NULL,\n event_type TEXT,\n embedding VECTOR(768)\n) SHARD BY tenant_id;\n```\n\nAll rows with the same `tenant_id` are guarante" + }, + { + "file": "pages/config-auth.md", + "title": "Authentication", + "section-id": "configuration", + "keywords": "authentication, API keys, mTLS, RBAC, SSO, pg_hba, security", + "description": "Configuring NeuralDB authentication \u2014 API keys, mTLS, role-based access control, and SSO", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Authentication\n\nNeuralDB supports multiple authentication methods, from simple password auth to mutual TLS and enterprise SSO. This page explains how to configure each.\n\n## Host-Based Authentication (pg_hba.conf)\n\nThe `pg_hba.conf` file controls which clients can connect and how they must authenticate. It is evaluated top-to-bottom and the first matching rule applies.\n\n```\n# pg_hba.conf\n# FORMAT: TYPE DATABASE USER ADDRESS METHOD\n\n# Local Unix socket connections (no password needed for postgres superuser)\nlocal all neuraldb trust\nlocal all all md5\n\n# IPv4 connections from local network\nhost all all 127.0.0.1/32 scram-sha-256\nhost all all 10.0.0.0/8 scram-sha-256\n\n# Reject all other connections\nhost all all 0.0.0.0/0 reject\n```\n\nReload after changes:\n\n```sql\nSELECT pg_reload_conf();\n```\n\n## Authentication Methods\n\n| Method | Security | Use case |\n|--------|---------|---------|\n| `trust` | None | Local socket, trusted environments |\n| `md5` | Weak | Legacy compatibility |\n| `scram-sha-256` | Strong (default) | Standard password auth |\n| `cert` | Very strong | Mutual TLS authentication |\n| `ldap` | Strong | Enterprise LDAP/AD integration |\n| `radius` | Strong | RADIUS server |\n| `gss` | Strong | Kerberos |\n| `jwt` | Strong | Token-based auth |\n\n### Enabling scram-sha-256\n\n```ini\n# neuraldb.conf\npassword_encryption = scram-sha-256\n```\n\nSet passwords for users:\n\n```sql\nCREATE USER app_user WITH PASSWORD 'secure-random-password';\nALTER USER app_user PASSWORD 'new-secure-password';\n```\n\n## Role-Based Access Control (RBAC)\n\n### Creating Roles\n\n```sql\n-- Application read-only role\nCREATE ROLE app_readonly;\nGRANT CONNECT ON DATABASE myapp TO app_readonly;\nGRANT USAGE ON SCHEMA public TO app_readonly;\nGRANT SELECT ON ALL TABLES IN SCHEMA public TO app_readonly;\nALTER DEFAULT PRIVILEGES IN SCHEMA public\n GRANT SELECT ON TABLES TO app_readonly;\n\n-- Application read-write role\nCREATE ROLE app_readwrite;\nGRANT app_readonly TO app_readwrite;\nGRANT INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO app_readwrite;\nALTER DEFAULT PRIVILEGES IN SCHEMA public\n GRANT INSERT, UPDATE, DELETE ON TABLES TO app_readwrite;\n\n-- Vector operations role (needed for ANN searches)\nCREATE ROLE vector_user;\nGRANT USAGE ON SCHEMA public TO vector_user;\nGRANT SELECT ON ALL TABLES IN SCHEMA public TO vector_user;\nGRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO vector_user;\n\n-- Application user\nCREATE USER my_app WITH PASSWORD 'app-password';\nGRANT app_readwrite TO my_app;\nGRANT vector_user TO my_app;\n```\n\n### Row-Level Security (RLS)\n\nFor multi-tenant applications, use Row-Level Security to enforce tenant isolation at the database level:\n\n```sql\nALTER TABLE documents ENABLE ROW LEVEL SECURITY;\n\n-- Tenants can only see their own documents\nCREATE POLICY tenant_isolation ON documents\n USING (tenant_id = current_setting('app.tenant_id')::UUID);\n\n-- Admin can see everything\nCREATE POLICY admin_access ON documents\n TO admin_role\n USING (true);\n```\n\nIn your application, set the tenant context before querying:\n\n```python\nwith connection.cursor() as cursor:\n cursor.execute(\"SET app.tenant_id = %s\", [tenant_id])\n cursor.execute(\"SELECT * FROM documents ORDER BY embedding <=> %s LIMIT 10\", [embedding])\n```\n\n## API Key Authentication\n\nNeuralDB supports an API key table for token-based authentication \u2014 useful for microservices and CI pipelines that cannot use password auth:\n\n```sql\n-- Enable the API key extension\nCREATE EXTENSION neuraldb_apikeys;\n\n-- Create an API key for a service account\nSELECT neuraldb_apikeys.create_key(\n label => 'my-service',\n role => 'app_readwrite'\n);\n-- Returns: ndb_live_abcdefghij123456...\n```\n\nClients authenticate by passing the key in the connection string:\n\n```\npostgresql://apikey:ndb_live_abc123@host:5432/mydb\n```\n\nRevoke a key:\n\n```sql\nSELECT neuraldb_apikeys.revoke_key('ndb_live_abc123');\n```\n\n## Mutual TLS (mTLS)\n\nFor the highest security, require clients to present a certificate signed by your CA.\n\n### 1. Generate a CA\n\n```bash\n# Generate CA key and certificate\nopenssl genrsa -out ca.key 4096\nopenssl req -new -x509 -key ca.key -out ca.crt -days 3650 \\\n -subj \"/CN=NeuralDB CA/O=My Org\"\n```\n\n### 2. Issue a Server Certificate\n\n```bash\nopenssl genrsa -out server.key 2048\nopenssl req -new -key server.key -out server.csr \\\n -subj \"/CN=neuraldb.example.com\"\nopenssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key \\\n -CAcreateserial -out server.crt -days 365\n```\n\n### 3. Configure NeuralDB\n\n```ini\n# neuraldb.conf\nssl = on\nssl_cert_file = '/etc/neuraldb/ssl/server.crt'\nssl_key_file = '/etc/neuraldb/ssl/server.key'\nssl_ca_file = '/etc/neuraldb/ssl/ca.crt'\nssl_crl_file = '/etc/neuraldb/ssl/crl.pem' # optional certificate revocation list\n```\n\n### 4. Require Client Certificates\n\n```\n# pg_hba.conf\nhostssl all all 0.0.0.0/0 cert clientcert=verify-full\n```\n\n## LDAP / Active Directory\n\n```\n# pg_hba.conf\nhost all a" + }, + { + "file": "pages/config-replication.md", + "title": "Replication", + "section-id": "configuration", + "keywords": "replication, primary, replica, streaming replication, multi-region, consistency", + "description": "Configuring NeuralDB replication \u2014 primary/replica setup, multi-region, and consistency levels", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Replication\n\nNeuralDB replication is based on streaming replication: the primary continuously ships WAL records to replicas, which apply them in real time. This page explains how to set up and configure replication.\n\n## Prerequisites\n\n- The primary must have `wal_level = replica` or higher\n- `max_wal_senders` must be greater than the number of replicas\n- A replication user must exist\n\n## Setting Up a Primary\n\nConfigure `neuraldb.conf` on the primary:\n\n```ini\n# neuraldb.conf (primary)\nwal_level = replica\nmax_wal_senders = 10\nmax_replication_slots = 10\nwal_keep_size = 1GB\nhot_standby_feedback = on # prevents primary from vacuuming rows still needed by replicas\n```\n\nCreate a replication user:\n\n```sql\nCREATE USER replicator WITH REPLICATION PASSWORD 'repl-password';\n```\n\nAllow the replica to connect:\n\n```\n# pg_hba.conf (primary)\nhost replication replicator replica-ip/32 scram-sha-256\n```\n\n## Setting Up a Replica\n\nOn the replica server, use `pg_basebackup` to clone the primary:\n\n```bash\n# On the replica server\npg_basebackup \\\n --host=primary.example.com \\\n --port=5432 \\\n --username=replicator \\\n --pgdata=/var/lib/neuraldb/data \\\n --wal-method=stream \\\n --checkpoint=fast \\\n --progress \\\n --write-recovery-conf\n```\n\nThe `--write-recovery-conf` flag creates a `standby.signal` file and writes connection info to `postgresql.auto.conf`, which tells NeuralDB to start in standby mode.\n\nConfigure `neuraldb.conf` on the replica:\n\n```ini\n# neuraldb.conf (replica)\nhot_standby = on # allow read queries\nhot_standby_feedback = on # send feedback to primary\nwal_receiver_timeout = 60s\nrecovery_min_apply_delay = 0 # apply WAL immediately (increase for delayed replicas)\n```\n\nStart the replica:\n\n```bash\nsystemctl start neuraldb\n```\n\nVerify replication is working:\n\n```sql\n-- On the primary\nSELECT client_addr, state, sent_lsn, write_lsn, flush_lsn, replay_lsn,\n (sent_lsn - replay_lsn) AS replication_lag_bytes\nFROM pg_stat_replication;\n```\n\n## Replication Slots\n\nReplication slots ensure the primary retains WAL until the replica has consumed it. This prevents the replica from falling too far behind, but also prevents WAL from being cleaned up if the replica disconnects.\n\n```sql\n-- Create a replication slot\nSELECT pg_create_physical_replication_slot('replica_1');\n\n-- List slots and their lag\nSELECT slot_name, active, pg_size_pretty(pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn)) AS lag\nFROM pg_replication_slots;\n\n-- Drop a slot (do this if a replica is permanently removed)\nSELECT pg_drop_replication_slot('replica_1');\n```\n\n**Warning:** Monitor slot lag. An inactive slot with large lag will cause unbounded WAL accumulation and can fill your disk.\n\n## Synchronous Replication\n\nBy default, replication is asynchronous \u2014 the primary does not wait for replicas to acknowledge writes. For zero data loss, configure synchronous replication:\n\n```ini\n# neuraldb.conf (primary)\nsynchronous_standby_names = 'FIRST 1 (replica1, replica2)'\n# ^ Wait for at least 1 of the listed standbys to acknowledge each commit\n```\n\nModes:\n- `FIRST n (list)` \u2014 wait for the first n standbys in the list\n- `ANY n (list)` \u2014 wait for any n standbys from the list\n- `*` \u2014 wait for all standbys\n\nPer-transaction override:\n\n```sql\nSET synchronous_commit = 'local'; -- this transaction doesn't wait for replicas\n```\n\n## Multi-Region Replication\n\nFor global deployments, replicate to remote regions. The configuration is identical to local replication, but network latency affects synchronous commit performance.\n\nRecommended approach for multi-region:\n\n```\nPrimary (us-east-1)\n\u251c\u2500\u2500 Sync replica (us-east-1-az2) \u2190 HA within region, ~2ms latency\n\u251c\u2500\u2500 Async replica (eu-west-1) \u2190 EU reads, ~80ms latency\n\u2514\u2500\u2500 Async replica (ap-northeast-1) \u2190 APAC reads, ~170ms latency\n```\n\n```ini\n# Synchronous only within region; async to remote regions\nsynchronous_standby_names = 'FIRST 1 (local_replica)'\n```\n\nConfigure the remote replicas with a `primary_conninfo` pointing to the primary:\n\n```ini\n# standby.signal (on replica)\nprimary_conninfo = 'host=primary.us-east-1.example.com port=5432 user=replicator password=repl-password sslmode=require'\n```\n\n## Failover\n\nNeuralDB does not include automatic failover out of the box. Use one of:\n\n- **Patroni** \u2014 industry-standard HA manager for PostgreSQL-compatible databases\n- **NeuralDB HA Operator** \u2014 Kubernetes operator with automatic failover (see [Kubernetes docs](install-kubernetes.md))\n- **repmgr** \u2014 lightweight failover manager\n\nManual failover:\n\n```bash\n# On the replica that will become the new primary\nneuraldb-cli -c \"SELECT pg_promote();\"\n```\n\nAfter promotion, update `primary_conninfo` on all other replicas to point to the new primary.\n\n## Monitoring Replication\n\n```sql\n-- Replication lag in bytes and seconds\nSELECT client_addr, state,\n pg_size_pretty(sent_lsn - replay_lsn) AS lag_bytes,\n now() - pg_last_xact_replay_timestamp() AS lag_time\nFROM pg_stat_replication;\n\n-- On a replica: chec" + }, + { + "file": "pages/config-server.md", + "title": "Server Config", + "section-id": "configuration", + "keywords": "neuraldb.conf, server configuration, settings, parameters, tuning", + "description": "Complete reference for neuraldb.conf \u2014 all server configuration settings explained", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Server Config\n\nNeuralDB is configured through `neuraldb.conf`, a key-value text file. This page documents all configuration parameters.\n\n## Locating the Config File\n\n```bash\n# Show the active config file path\nneuraldb-cli -c \"SHOW config_file;\"\n```\n\nDefault locations:\n- Linux: `/etc/neuraldb/neuraldb.conf`\n- macOS Homebrew: `$(brew --prefix)/etc/neuraldb/neuraldb.conf`\n- Docker: `/var/lib/neuraldb/data/neuraldb.conf`\n\nChanges to `neuraldb.conf` require a reload (for most parameters) or a restart:\n\n```bash\n# Reload without restart (applies most parameters)\nneuraldb-cli -c \"SELECT pg_reload_conf();\"\n\n# Full restart (required for listen_addresses, port, shared_buffers, etc.)\nsystemctl restart neuraldb\n```\n\n## Connection Settings\n\n```ini\n# Network interface to listen on\n# '*' = all interfaces, 'localhost' = local only\nlisten_addresses = '*'\n\n# TCP port\nport = 5432\n\n# Maximum simultaneous connections\n# Each connection uses ~5 MB of memory\nmax_connections = 100\n\n# Unix domain socket directory (Linux/macOS only)\nunix_socket_directories = '/var/run/neuraldb'\n\n# Superuser reserved connections\n# Reserves this many connections exclusively for superusers\nsuperuser_reserved_connections = 3\n```\n\n## Memory Settings\n\nThese are the most impactful parameters for performance.\n\n```ini\n# Page cache for relational data (row store)\n# Recommended: 25% of available RAM\nshared_buffers = 4GB\n\n# Memory for HNSW vector indexes\n# Recommended: 40-60% of available RAM for vector-heavy workloads\n# Must be large enough to fit all active HNSW graphs\nvector_buffer = 8GB\n\n# Per-query working memory (sorts, hash joins)\n# Recommended: 64MB\u2013256MB for OLTP, more for analytical queries\n# Be conservative: (max_connections \u00d7 work_mem) should fit in RAM\nwork_mem = 128MB\n\n# Memory for DDL maintenance (CREATE INDEX, VACUUM, etc.)\nmaintenance_work_mem = 2GB\n\n# Shared memory for parallel query workers\nparallel_query_mem = 512MB\n```\n\n## Vector Settings\n\n```ini\n# Default HNSW ef_search parameter (candidates evaluated per query)\n# Higher = better recall, slower queries\nhnsw.ef_search = 40\n\n# Maximum number of vectors per shard before auto-splitting\nvector_shard_size = 10000000\n\n# Enable approximate nearest neighbour by default (true = HNSW index)\n# Set to 'exact' to always use exact search (ignores vector indexes)\nvector_scan = 'approximate'\n\n# Number of parallel threads for HNSW index builds\nhnsw.build_threads = 4\n\n# Compression algorithm for stored vector data\n# 'none', 'lz4', 'scalar_quantize' (lossy but 4\u00d7 smaller)\nvector_compression = 'lz4'\n```\n\n## WAL Settings\n\n```ini\n# WAL logging level\n# 'minimal': minimum for crash recovery\n# 'replica': enables streaming replication (default)\n# 'logical': enables logical replication and CDC\nwal_level = replica\n\n# WAL segment size (change requires initdb)\nwal_segment_size = 128MB\n\n# Maximum number of concurrent WAL sender processes\nmax_wal_senders = 10\n\n# Number of WAL segments to keep for standby catching up\nwal_keep_size = 1GB\n\n# Synchronise WAL to disk before acknowledging commit\n# 'on': safest; 'local': only local disk; 'off': async (fastest, tiny durability risk)\nsynchronous_commit = on\n\n# Interval between WAL checkpoints\ncheckpoint_timeout = 5min\ncheckpoint_completion_target = 0.9\nmax_wal_size = 4GB\n```\n\n## Replication Settings\n\n```ini\n# Comma-separated list of standby names that must acknowledge writes\n# Leave empty for asynchronous replication\nsynchronous_standby_names = ''\n\n# Maximum lag allowed before primary throttles writes\nmax_standby_lag = 30s\n\n# Hot standby: allow reads on replicas\nhot_standby = on\n```\n\n## Query Planner\n\n```ini\n# Estimated cost of a random page fetch (tune based on SSD vs HDD)\n# Lower values favour index scans; higher values favour sequential scans\nrandom_page_cost = 1.1 # NVMe SSD (default is 4.0 for HDD)\n\n# Effective size of the disk cache (affects planner estimates)\neffective_cache_size = 24GB # ~75% of RAM\n\n# Enable parallel query\nmax_parallel_workers_per_gather = 4\nmax_parallel_workers = 8\nmax_worker_processes = 16\n\n# Hybrid query planner behaviour\n# 'auto': planner decides pre-filter vs post-filter\n# 'pre-filter': always pre-filter\n# 'post-filter': always post-filter\nvector_hybrid_strategy = 'auto'\n```\n\n## Logging\n\n```ini\n# Log destination\nlog_destination = 'stderr' # 'stderr', 'csvlog', 'jsonlog', 'syslog'\n\n# Minimum severity to log\n# 'DEBUG5' (verbose) \u2192 'INFO' \u2192 'WARNING' \u2192 'ERROR' \u2192 'FATAL'\nlog_min_messages = WARNING\n\n# Log all SQL statements with duration above this threshold (ms)\n# -1 = disable; 0 = log everything; 250 = log slow queries only\nlog_min_duration_statement = 250\n\n# Log query parameters\nlog_parameters = off\n\n# Log connection events\nlog_connections = on\nlog_disconnections = on\n\n# Log lock waits longer than this (ms)\ndeadlock_timeout = 1s\nlog_lock_waits = on\n```\n\n## Full Configuration Example\n\n```ini\n# neuraldb.conf \u2014 Production settings for a 32 vCPU / 128 GB server\n\nlisten_addresses = '*'\nport = 5432\nmax_connections = 500\nsuperuser_reserved_connections" + }, + { + "file": "pages/config-storage.md", + "title": "Storage Config", + "section-id": "configuration", + "keywords": "storage, memory, disk, SSD tiers, compression, tablespace, IOPS", + "description": "Configuring NeuralDB storage \u2014 memory tiers, disk layout, compression, and tablespaces", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Storage Config\n\nNeuralDB's performance depends heavily on storage configuration. This page covers how to optimise disk layout, configure memory tiers, enable compression, and use tablespaces for data tiering.\n\n## Disk Layout Recommendations\n\nSeparate the data directory, WAL, and vector files onto different physical disks (or at least different volumes with guaranteed IOPS):\n\n| Directory | Recommended storage | IOPS requirement |\n|-----------|--------------------|--------------------|\n| `$DATADIR/base/` | NVMe SSD | High random read/write |\n| `$DATADIR/wal/` | NVMe SSD (separate) | High sequential write |\n| `$DATADIR/vectors/` | NVMe SSD or RAM disk | High random read |\n| `$DATADIR/archive/` | HDD or object storage | Low (sequential write) |\n\nConfigure separate paths in `neuraldb.conf`:\n\n```ini\n# Separate WAL onto a dedicated volume\nwal_directory = '/mnt/nvme-wal/neuraldb/wal'\n\n# Separate vector storage\nvector_data_directory = '/mnt/nvme-fast/neuraldb/vectors'\n\n# Archive destination for WAL shipping\narchive_status_directory = '/var/lib/neuraldb/archive_status'\n```\n\n## Memory Tiers\n\nNeuralDB has three distinct memory pools:\n\n### shared_buffers (Row Store Cache)\n\nThe page cache for relational data. Sized at 25% of available RAM for a dedicated database server:\n\n```ini\nshared_buffers = 32GB # for a 128 GB server\n```\n\n### vector_buffer (Vector Index Cache)\n\nHolds the HNSW graph in memory. The entire active HNSW graph must fit in `vector_buffer` for optimal performance. When the graph doesn't fit, NeuralDB falls back to disk-based graph traversal, which is 10\u201350\u00d7 slower.\n\nCalculate the required size:\n\n```\nvector_buffer = num_vectors \u00d7 dimensions \u00d7 4 bytes \u00d7 hnsw_overhead_factor\n```\n\nWhere `hnsw_overhead_factor` \u2248 1.3 for default HNSW parameters (m=16).\n\n```sql\n-- Check current vector index memory usage\nSELECT table_name, index_name,\n pg_size_pretty(hnsw_graph_size_bytes) AS graph_memory,\n pg_size_pretty(vector_data_size_bytes) AS data_size\nFROM neuraldb_stat_vector_indexes;\n```\n\n### work_mem (Per-Query Buffer)\n\nUsed for in-memory sorts, hash joins, and bitmap operations. Set conservatively \u2014 each query can allocate multiple `work_mem` buffers:\n\n```ini\n# For 200 connections with typical 2 buffers per query:\n# Max memory consumption: 200 \u00d7 2 \u00d7 128MB = 51 GB\nwork_mem = 128MB\n```\n\nOverride per-session for analytical queries:\n\n```sql\nSET work_mem = '2GB';\nSELECT ... complex analytical query ...\n```\n\n## Compression\n\n### Row Store Compression\n\nNeuralDB compresses SSTables on disk. Choose the algorithm based on your priorities:\n\n```ini\n# Compression algorithm for row data\n# 'none': no compression (fastest reads/writes, most disk)\n# 'lz4': fast, moderate compression ratio (~2-3\u00d7) \u2014 default\n# 'zstd': slower compression, better ratio (~3-5\u00d7)\n# 'zstd-9': high compression for archival (slow, ~6-8\u00d7)\nstorage_compression = lz4\n\n# Compression level (for zstd only), 1-19\nstorage_compression_level = 3\n```\n\n### Vector Compression\n\n```ini\n# Vector data compression\n# 'none': full precision, largest storage\n# 'lz4': fast, minimal precision loss\n# 'scalar_quantize': reduce to 8-bit (4\u00d7 smaller, ~1% recall loss)\n# 'product_quantize': very high compression, higher recall loss\nvector_compression = lz4\n```\n\nTo enable scalar quantisation for a specific index:\n\n```sql\nCREATE INDEX ON documents\nUSING hnsw (embedding vector_cosine_ops)\nWITH (quantization = 'scalar');\n```\n\nScalar-quantised indexes use 4\u00d7 less memory and may be faster due to better cache utilisation, at a typical recall cost of 0.5\u20132%.\n\n## Tablespaces\n\nUse tablespaces to store different tables or indexes on different volumes:\n\n```sql\n-- Create tablespaces pointing to different mount points\nCREATE TABLESPACE fast_ssd LOCATION '/mnt/nvme-fast';\nCREATE TABLESPACE bulk_hdd LOCATION '/mnt/hdd-storage';\n\n-- Create a table on fast SSD\nCREATE TABLE hot_documents (\n id UUID PRIMARY KEY,\n content TEXT,\n embedding VECTOR(1536)\n) TABLESPACE fast_ssd;\n\n-- Move an index to a specific tablespace\nCREATE INDEX ON hot_documents USING hnsw (embedding vector_cosine_ops)\nTABLESPACE fast_ssd;\n\n-- Move old partitions to cheaper storage\nALTER TABLE documents_2024_q1 SET TABLESPACE bulk_hdd;\n```\n\n## Table Partitioning\n\nPartition large tables by time or tenant to improve query performance and manageability:\n\n```sql\n-- Partition documents by month\nCREATE TABLE documents (\n id UUID NOT NULL DEFAULT gen_random_uuid(),\n content TEXT,\n embedding VECTOR(1536),\n tenant_id UUID,\n created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()\n) PARTITION BY RANGE (created_at);\n\nCREATE TABLE documents_2026_01\n PARTITION OF documents\n FOR VALUES FROM ('2026-01-01') TO ('2026-02-01')\n TABLESPACE fast_ssd;\n\nCREATE TABLE documents_2025_archive\n PARTITION OF documents\n FOR VALUES FROM ('2025-01-01') TO ('2026-01-01')\n TABLESPACE bulk_hdd;\n```\n\n## VACUUM and Autovacuum\n\nNeuralDB inherits PostgreSQL's MVCC-based VACUUM system:\n\n```ini\n# Autovacuum settings\nautovacuum = on\nautovacuum_naptime = 1m" + }, + { + "file": "pages/index.md", + "title": "What is NeuralDB?", + "section-id": "overview", + "keywords": "NeuralDB, introduction, AI database, vector database, overview", + "description": "What NeuralDB is, its key use cases, and a high-level architecture overview", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# What is NeuralDB?\n\n![NeuralDB Platform](assets/images/hero.jpg)\n\nNeuralDB is an AI-native database that unifies vector storage, semantic search, and relational data management in a single, horizontally scalable system. It is designed for applications that need to combine structured data queries with the semantic understanding that modern AI models provide.\n\nTraditional databases store and retrieve data based on exact matches and range queries. NeuralDB adds a third dimension: **semantic proximity**. You can ask \"find the 20 products most similar to this description\" at the same time as \"where inventory > 0 and price < 100\" \u2014 in a single, atomic query with full ACID guarantees.\n\n## Why NeuralDB Exists\n\nThe AI application stack of 2024\u20132026 exposed a fundamental tension in data architecture. Teams building retrieval-augmented generation (RAG) systems, recommendation engines, and semantic search needed:\n\n- A **vector database** (Pinecone, Weaviate, Qdrant) for embedding storage and similarity search\n- A **relational database** (PostgreSQL, MySQL) for structured metadata and transactions\n- A **synchronisation layer** to keep the two in sync \u2014 a constant source of bugs and operational overhead\n\nNeuralDB replaces all three with a single system. Vectors and relational data share the same storage engine, the same transaction log, and the same query planner. There is no synchronisation problem because there is no synchronisation needed.\n\n## Key Capabilities\n\n### Hybrid Vector + Relational Queries\n\n```sql\nSELECT id, name, price, SIMILARITY(embedding, :query_embedding) AS score\nFROM products\nWHERE category = 'electronics'\n AND price BETWEEN 50 AND 500\n AND stock > 0\nORDER BY score DESC\nLIMIT 10;\n```\n\nThis query uses a vector index for semantic ranking and a B-tree index for the relational filters, with the query planner choosing the optimal execution order.\n\n### Multiple Vector Index Types\n\nNeuralDB supports three similarity metrics out of the box:\n\n| Metric | Use case |\n|--------|----------|\n| Cosine similarity | Text embeddings, normalised vectors |\n| Dot product | Recommendation systems (unnormalised) |\n| Euclidean (L2) | Image embeddings, spatial data |\n\n### Full ACID Transactions\n\nUnlike most vector databases, NeuralDB provides full ACID guarantees \u2014 including transactional upserts of both the relational row and the vector embedding simultaneously.\n\n### Automatic Embedding Updates\n\nConfigure NeuralDB to call an embedding API whenever a text column changes:\n\n```sql\nALTER TABLE documents\nADD COLUMN embedding VECTOR(1536)\nGENERATED ALWAYS AS EMBEDDING OF content\nUSING openai_ada_002;\n```\n\nNeuralDB handles the embedding pipeline automatically \u2014 no application-level embedding code required.\n\n### Multi-Modal Vectors\n\nStore and query vectors from any modality \u2014 text, image, audio, or custom \u2014 in the same table:\n\n```sql\nCREATE TABLE media_items (\n id UUID PRIMARY KEY,\n title TEXT,\n text_embedding VECTOR(1536),\n image_embedding VECTOR(512),\n created_at TIMESTAMP DEFAULT NOW()\n);\n```\n\n## Use Cases\n\n**Retrieval-Augmented Generation (RAG)** \u2014 Store your document corpus with embeddings. Query the most relevant chunks in a single round-trip and inject them into your LLM prompt.\n\n**Semantic Product Search** \u2014 Replace keyword search with semantic search. Find products matching \"comfortable running shoes for flat feet\" even if those exact words don't appear in any product description.\n\n**Recommendation Engines** \u2014 Store user preference vectors alongside item vectors. Compute collaborative-filtering recommendations with a single NQL query.\n\n**Anomaly Detection** \u2014 Flag records whose vectors are distant from the cluster of \"normal\" data.\n\n**Duplicate Detection** \u2014 Find near-duplicate records across millions of rows using approximate nearest-neighbour (ANN) search.\n\n**Knowledge Graphs** \u2014 Store entity embeddings alongside relationship metadata for graph-enhanced retrieval.\n\n## NeuralDB vs Traditional Approaches\n\n| Capability | NeuralDB | Postgres + pgvector | Pinecone + Postgres |\n|-----------|---------|---------------------|---------------------|\n| Vector search | Native, HNSW | Extension, limited | Native |\n| Relational queries | Full SQL | Full SQL | None (separate DB) |\n| Hybrid queries | Single query | Single query | Application-layer join |\n| ACID transactions | Yes | Yes | Partial |\n| Horizontal sharding | Built-in | Manual (Citus) | Managed |\n| Automatic embeddings | Yes | No | No |\n| Streaming ingestion | Yes | No | Partial |\n\n## Getting Started\n\nThe fastest way to try NeuralDB is with Docker:\n\n```bash\ndocker run -p 5432:5432 -e NEURALDB_PASSWORD=mypassword neuraldb/neuraldb:latest\n```\n\nConnect with any PostgreSQL-compatible client:\n\n```bash\npsql -h localhost -U neuraldb -d neuraldb\n```\n\nThen read the [Core Concepts](concepts.md) to understand the NeuralDB data model, or jump to [NQL Basics](nql-basics.md) to start writing queries." + }, + { + "file": "pages/install-cloud.md", + "title": "Cloud Managed", + "section-id": "installation", + "keywords": "cloud, managed, NeuralDB Cloud, regions, tiers, SaaS", + "description": "Setting up NeuralDB Cloud \u2014 the fully managed service with global regions and flexible tiers", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Cloud Managed\n\nNeuralDB Cloud is the fully managed version of NeuralDB. It handles provisioning, patching, backups, monitoring, and scaling \u2014 so you can focus on building your application rather than managing database infrastructure.\n\n## Getting Started\n\n### 1. Create an Account\n\nSign up at [cloud.neuraldb.io](https://cloud.neuraldb.io). You can authenticate with Google, GitHub, or an email address.\n\n### 2. Create a Cluster\n\nClick **New Cluster** and configure:\n\n- **Region**: choose the cloud region closest to your application servers\n- **Tier**: select based on your workload requirements (see tier comparison below)\n- **Storage**: initial storage allocation (can be scaled later)\n- **High Availability**: enable for production workloads\n\n### 3. Connect\n\nOnce the cluster is provisioned (typically under 3 minutes), your connection string appears in the dashboard:\n\n```\npostgresql://neuraldb:[password]@[cluster-id].cloud.neuraldb.io:5432/[database]?sslmode=require\n```\n\nUse this with any PostgreSQL-compatible driver or psql:\n\n```bash\npsql \"postgresql://neuraldb:mypassword@abc123.cloud.neuraldb.io:5432/mydb?sslmode=require\"\n```\n\n## Available Regions\n\nNeuralDB Cloud is available in the following regions:\n\n| Region | Cloud Provider | Availability |\n|--------|---------------|-------------|\n| us-east-1 (N. Virginia) | AWS | GA |\n| us-west-2 (Oregon) | AWS | GA |\n| eu-west-1 (Ireland) | AWS | GA |\n| eu-central-1 (Frankfurt) | AWS | GA |\n| ap-northeast-1 (Tokyo) | AWS | GA |\n| ap-southeast-1 (Singapore) | AWS | GA |\n| us-central1 (Iowa) | GCP | Beta |\n| europe-west4 (Netherlands) | GCP | Beta |\n| eastus (Virginia) | Azure | Beta |\n\nMulti-region replication (primary in one region, read replicas in others) is available on Business and Enterprise tiers.\n\n## Pricing Tiers\n\n### Starter\n\nFree tier for development and experimentation.\n\n| Resource | Limit |\n|---------|-------|\n| Storage | 5 GB |\n| Vector dimensions | Up to 1536 |\n| Max connections | 10 |\n| PITR | No |\n| HA | No |\n| SLA | No |\n\nThe Starter tier automatically suspends after 7 days of inactivity and resumes on the next connection (cold start: ~30 seconds).\n\n### Developer\n\n$29/month \u2014 for side projects and pre-production environments.\n\n| Resource | Limit |\n|---------|-------|\n| vCPU | 2 dedicated |\n| RAM | 8 GB |\n| Storage | 100 GB NVMe SSD |\n| Connections | 100 |\n| PITR | 7 days |\n| HA | No |\n\n### Business\n\n$199/month \u2014 for production workloads.\n\n| Resource | Limit |\n|---------|-------|\n| vCPU | 8 dedicated |\n| RAM | 32 GB |\n| Storage | 500 GB NVMe SSD (expandable) |\n| Connections | 500 |\n| PITR | 30 days |\n| HA | Yes (1 standby) |\n| Read replicas | Up to 3 |\n| SLA | 99.95% |\n\n### Business Plus\n\n$599/month \u2014 for large-scale production workloads.\n\n| Resource | Limit |\n|---------|-------|\n| vCPU | 32 dedicated |\n| RAM | 128 GB |\n| Storage | 2 TB NVMe SSD (expandable) |\n| Connections | 2,000 |\n| PITR | 30 days |\n| HA | Yes (2 standbys) |\n| Read replicas | Up to 10 |\n| Multi-region replicas | Yes |\n| SLA | 99.99% |\n\n### Enterprise\n\nCustom pricing \u2014 for mission-critical applications with specific compliance requirements.\n\n- Dedicated infrastructure (no multi-tenancy)\n- Custom SLAs\n- Private endpoints (AWS PrivateLink, GCP Private Service Connect)\n- SOC 2 Type II, HIPAA, and ISO 27001 compliance\n- Dedicated support with SLA\n\n## Connecting from Your Application\n\n### Connection Pooling\n\nNeuralDB Cloud includes PgBouncer-based connection pooling. Use the pooler endpoint for serverless and short-lived connections:\n\n```\npostgresql://neuraldb:[password]@[cluster-id]-pooler.cloud.neuraldb.io:5432/[database]\n```\n\n| Connection type | Endpoint suffix | Use for |\n|----------------|----------------|---------|\n| Direct | (none) | Long-lived connections, COPY operations |\n| Transaction pooling | `-pooler` | Serverless, short-lived connections |\n| Session pooling | `-session-pooler` | Prepared statements |\n\n### IP Allow-listing\n\nRestrict access to specific IP ranges in the **Security** tab of your cluster dashboard. Enter your application servers' CIDR ranges (e.g., `10.0.0.0/8` for private VPC, or specific public IPs).\n\n### SSL/TLS\n\nAll connections require TLS. Download the cluster CA certificate from the dashboard and verify it in your connection string:\n\n```\nsslmode=verify-full&sslrootcert=/path/to/ca.pem\n```\n\n## Monitoring and Alerting\n\nNeuralDB Cloud includes a built-in monitoring dashboard with:\n\n- Query throughput and latency percentiles (p50, p95, p99)\n- Connection count\n- Storage usage\n- Vector index size\n- Replication lag\n\nConfigure alerts for:\n- Storage > 80% full\n- Average query latency > 500ms\n- Replication lag > 30s\n- Failed connections\n\nMetrics are also available via the NeuralDB Cloud API for ingestion into your own monitoring stack (Datadog, Grafana Cloud, New Relic).\n\n## Branching (Database Branches)\n\nNeuralDB Cloud supports **branching** \u2014 create an instant copy-on-write clone of your production database for development, testing, or migrations:\n\n```bash\nneuraldb-cl" + }, + { + "file": "pages/install-docker.md", + "title": "Docker Install", + "section-id": "installation", + "keywords": "Docker, install, docker run, docker-compose, volumes, container", + "description": "Installing NeuralDB using Docker \u2014 single container and docker-compose setups", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Docker Install\n\nDocker is the fastest way to run NeuralDB locally or in a single-server deployment. NeuralDB's official Docker image is published to Docker Hub as `neuraldb/neuraldb`.\n\n## Quick Start\n\nRun a single NeuralDB instance:\n\n```bash\ndocker run -d \\\n --name neuraldb \\\n -p 5432:5432 \\\n -e NEURALDB_PASSWORD=mypassword \\\n -e NEURALDB_DB=mydb \\\n -v neuraldb_data:/var/lib/neuraldb/data \\\n neuraldb/neuraldb:latest\n```\n\nConnect with psql:\n\n```bash\npsql -h localhost -p 5432 -U neuraldb -d mydb\n# Password: mypassword\n```\n\n## Environment Variables\n\n| Variable | Default | Description |\n|----------|---------|-------------|\n| `NEURALDB_PASSWORD` | required | Password for the `neuraldb` superuser |\n| `NEURALDB_USER` | `neuraldb` | Superuser username |\n| `NEURALDB_DB` | `neuraldb` | Default database name |\n| `NEURALDB_PORT` | `5432` | TCP port |\n| `NEURALDB_MAX_CONNECTIONS` | `100` | Maximum concurrent connections |\n| `NEURALDB_SHARED_BUFFERS` | `256MB` | Row store page cache |\n| `NEURALDB_VECTOR_BUFFER` | `512MB` | Vector index memory |\n| `NEURALDB_WAL_LEVEL` | `replica` | WAL level (`minimal`, `replica`, `logical`) |\n\n## Available Tags\n\n| Tag | Description |\n|-----|-------------|\n| `latest` | Latest stable release |\n| `1.0` | Specific major version |\n| `1.0.3` | Specific patch version |\n| `nightly` | Nightly build from main branch |\n| `1.0-alpine` | Alpine-based image (smaller, less glibc compat) |\n\n## Volumes\n\nNeuralDB stores data in `/var/lib/neuraldb/data` inside the container. Always mount a named volume or bind mount to persist data:\n\n```bash\n# Named volume (recommended)\ndocker volume create neuraldb_data\ndocker run -v neuraldb_data:/var/lib/neuraldb/data neuraldb/neuraldb:latest\n\n# Bind mount\ndocker run -v /srv/neuraldb:/var/lib/neuraldb/data neuraldb/neuraldb:latest\n```\n\nThe data directory includes:\n- `base/` \u2014 table and index data\n- `vectors/` \u2014 HNSW graph files and raw vector data\n- `wal/` \u2014 write-ahead log segments\n- `neuraldb.conf` \u2014 runtime configuration (editable)\n\n## docker-compose Setup\n\nA production-grade docker-compose file for NeuralDB with automatic backup:\n\n```yaml\n# docker-compose.yml\nversion: '3.9'\n\nservices:\n neuraldb:\n image: neuraldb/neuraldb:1.0\n container_name: neuraldb\n restart: unless-stopped\n ports:\n - \"127.0.0.1:5432:5432\" # bind to localhost only \u2014 use nginx for external access\n environment:\n NEURALDB_PASSWORD: ${NEURALDB_PASSWORD}\n NEURALDB_USER: ${NEURALDB_USER:-neuraldb}\n NEURALDB_DB: ${NEURALDB_DB:-neuraldb}\n NEURALDB_SHARED_BUFFERS: \"4GB\"\n NEURALDB_VECTOR_BUFFER: \"8GB\"\n NEURALDB_MAX_CONNECTIONS: \"200\"\n volumes:\n - neuraldb_data:/var/lib/neuraldb/data\n - neuraldb_wal:/var/lib/neuraldb/wal\n - ./neuraldb.conf:/etc/neuraldb/neuraldb.conf:ro # optional custom config\n shm_size: '2gb' # increase shared memory for large sort operations\n healthcheck:\n test: [\"CMD-SHELL\", \"pg_isready -U ${NEURALDB_USER:-neuraldb}\"]\n interval: 10s\n timeout: 5s\n retries: 5\n deploy:\n resources:\n limits:\n memory: 16G\n reservations:\n memory: 8G\n\n neuraldb-backup:\n image: neuraldb/neuraldb-backup:latest\n environment:\n NEURALDB_HOST: neuraldb\n NEURALDB_PASSWORD: ${NEURALDB_PASSWORD}\n S3_BUCKET: ${BACKUP_S3_BUCKET}\n S3_PREFIX: backups/neuraldb/\n SCHEDULE: \"0 2 * * *\" # 2am daily\n depends_on:\n neuraldb:\n condition: service_healthy\n\nvolumes:\n neuraldb_data:\n driver: local\n driver_opts:\n type: none\n o: bind\n device: /srv/neuraldb/data\n neuraldb_wal:\n driver: local\n driver_opts:\n type: none\n o: bind\n device: /srv/neuraldb/wal\n```\n\nStart with:\n\n```bash\necho \"NEURALDB_PASSWORD=$(openssl rand -base64 32)\" > .env\ndocker-compose up -d\n```\n\n## Initialisation Scripts\n\nPlace `.sql` or `.sh` scripts in `/docker-entrypoint-initdb.d/` to run them on first startup:\n\n```bash\ndocker run \\\n -v ./init-scripts:/docker-entrypoint-initdb.d:ro \\\n -e NEURALDB_PASSWORD=mypassword \\\n neuraldb/neuraldb:latest\n```\n\n```sql\n-- init-scripts/01-schema.sql\nCREATE TABLE documents (\n id UUID PRIMARY KEY DEFAULT gen_random_uuid(),\n content TEXT NOT NULL,\n embedding VECTOR(1536),\n created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()\n);\n\nCREATE INDEX ON documents USING hnsw (embedding vector_cosine_ops);\n```\n\n## Memory Tuning\n\nFor production, size the container memory based on your dataset:\n\n```\nRecommended memory = shared_buffers + vector_buffer + (max_connections \u00d7 work_mem) + OS overhead\n```\n\nFor a typical RAG application (5M documents, 1536 dimensions):\n- `vector_buffer` \u2248 5M \u00d7 1536 \u00d7 4B \u00d7 1.3 = ~40 GB\n- `shared_buffers` = 8 GB\n- `work_mem` \u00d7 connections = 128MB \u00d7 50 = 6.4 GB\n- **Total**: ~56 GB \u2014 provision a 64 GB container\n\n## Upgrading\n\n```bash\n# Pull the new image\ndocker pull neuraldb/neuraldb:1.1\n\n# Stop the current container (data is safe in the volume)\ndocker stop neuraldb && docker rm neuraldb" + }, + { + "file": "pages/install-kubernetes.md", + "title": "Kubernetes", + "section-id": "installation", + "keywords": "Kubernetes, Helm, StatefulSet, PVC, k8s, cluster, deployment", + "description": "Deploying NeuralDB on Kubernetes using the official Helm chart and StatefulSets", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Kubernetes\n\nThe recommended way to run NeuralDB on Kubernetes is via the official Helm chart. The chart deploys NeuralDB as a StatefulSet with persistent volume claims, and supports both standalone and high-availability configurations.\n\n## Prerequisites\n\n- Kubernetes 1.27+\n- Helm 3.x\n- A storage class that supports `ReadWriteOnce` PVCs (most cloud providers support this)\n- At least 4 CPU cores and 8 GB RAM per NeuralDB node\n\n## Installing the Helm Chart\n\n```bash\n# Add the NeuralDB Helm repository\nhelm repo add neuraldb https://charts.neuraldb.io\nhelm repo update\n\n# Create a namespace\nkubectl create namespace neuraldb\n\n# Install the chart\nhelm install neuraldb neuraldb/neuraldb \\\n --namespace neuraldb \\\n --set auth.password=mysecretpassword \\\n --set persistence.size=100Gi\n```\n\n## Chart Configuration\n\nCreate a `values.yaml` file for production settings:\n\n```yaml\n# values.yaml\n\nimage:\n repository: neuraldb/neuraldb\n tag: \"1.0\"\n pullPolicy: IfNotPresent\n\nauth:\n # Set via --set auth.password=... or a pre-existing secret\n existingSecret: \"\"\n secretKey: \"neuraldb-password\"\n\nreplicaCount: 1 # primary nodes (use 1 for standalone)\nreadReplicaCount: 2 # read replicas\n\nresources:\n requests:\n cpu: \"2\"\n memory: \"8Gi\"\n limits:\n cpu: \"8\"\n memory: \"32Gi\"\n\npersistence:\n enabled: true\n storageClass: \"fast-ssd\" # use a fast SSD storage class\n size: 500Gi\n walSize: 50Gi # separate PVC for WAL\n\nvectorBuffer: \"16Gi\" # memory for HNSW index\nsharedBuffers: \"8Gi\" # row store page cache\nmaxConnections: 200\n\nservice:\n type: ClusterIP\n port: 5432\n\n# High-availability configuration\nha:\n enabled: true\n replication:\n mode: synchronous # 'synchronous' or 'asynchronous'\n synchronousCommit: \"on\"\n\nbackup:\n enabled: true\n schedule: \"0 2 * * *\"\n s3:\n bucket: my-neuraldb-backups\n region: us-east-1\n existingSecret: aws-credentials\n\nmonitoring:\n enabled: true\n serviceMonitor:\n enabled: true # requires Prometheus Operator\n```\n\nApply the values:\n\n```bash\nhelm install neuraldb neuraldb/neuraldb \\\n --namespace neuraldb \\\n -f values.yaml \\\n --set auth.password=$(openssl rand -base64 32)\n```\n\n## StatefulSet Details\n\nThe chart deploys a `StatefulSet` with:\n\n- One pod per replica (primary + read replicas)\n- Two PVCs per pod: data volume and WAL volume\n- An init container that configures replication on startup\n\n```yaml\n# Example pod spec (simplified)\nspec:\n containers:\n - name: neuraldb\n image: neuraldb/neuraldb:1.0\n ports:\n - containerPort: 5432\n resources:\n requests:\n memory: \"8Gi\"\n cpu: \"2\"\n volumeMounts:\n - name: data\n mountPath: /var/lib/neuraldb/data\n - name: wal\n mountPath: /var/lib/neuraldb/wal\n livenessProbe:\n exec:\n command: [\"pg_isready\", \"-U\", \"neuraldb\"]\n initialDelaySeconds: 30\n periodSeconds: 10\n readinessProbe:\n exec:\n command: [\"pg_isready\", \"-U\", \"neuraldb\"]\n initialDelaySeconds: 5\n periodSeconds: 5\n```\n\n## Services\n\nThe chart creates three Kubernetes services:\n\n| Service | Type | Port | Description |\n|---------|------|------|-------------|\n| `neuraldb-primary` | ClusterIP | 5432 | Primary \u2014 reads + writes |\n| `neuraldb-replica` | ClusterIP | 5432 | Read replicas \u2014 reads only |\n| `neuraldb-headless` | Headless | 5432 | For StatefulSet pod discovery |\n\nConnect to the primary:\n\n```bash\nkubectl port-forward svc/neuraldb-primary 5432:5432 -n neuraldb\npsql -h localhost -U neuraldb\n```\n\n## Persistent Volume Claims\n\nEach pod gets two PVCs:\n\n```yaml\nvolumeClaimTemplates:\n- metadata:\n name: data\n spec:\n accessModes: [\"ReadWriteOnce\"]\n storageClassName: fast-ssd\n resources:\n requests:\n storage: 500Gi\n- metadata:\n name: wal\n spec:\n accessModes: [\"ReadWriteOnce\"]\n storageClassName: fast-ssd\n resources:\n requests:\n storage: 50Gi\n```\n\nUse a **fast-ssd** storage class (AWS `gp3`, GCP `pd-ssd`, Azure `Premium_LRS`) for the data and WAL volumes. Spinning disks are not supported in production.\n\n## Secrets Management\n\nStore the NeuralDB password in a Kubernetes secret:\n\n```bash\nkubectl create secret generic neuraldb-credentials \\\n --namespace neuraldb \\\n --from-literal=password=$(openssl rand -base64 32)\n```\n\nReference it in `values.yaml`:\n\n```yaml\nauth:\n existingSecret: neuraldb-credentials\n secretKey: password\n```\n\nFor larger installations, use an external secrets manager (HashiCorp Vault, AWS Secrets Manager) with the External Secrets Operator.\n\n## Scaling Read Replicas\n\nScale the number of read replicas without downtime:\n\n```bash\nhelm upgrade neuraldb neuraldb/neuraldb \\\n --namespace neuraldb \\\n --set readReplicaCount=4\n```\n\nThe new replica pods will join the replication stream automatically.\n\n## Upgrading\n\n```bash\nhelm repo update\nhelm upgrade neuraldb neuraldb/neuraldb \\\n --namespace neuraldb \\\n -f values.yaml \\\n --set auth.existingSecret=neuraldb-credentials\n```\n\nT" + }, + { + "file": "pages/install-local.md", + "title": "Local Development", + "section-id": "installation", + "keywords": "local, development, binary, homebrew, winget, install, macOS, Linux, Windows", + "description": "Installing NeuralDB locally for development using binaries, Homebrew, or winget", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Local Development\n\nFor local development, you can run NeuralDB as a native binary without Docker. This provides lower latency for development workflows and avoids container overhead.\n\n## System Requirements\n\n| Platform | Minimum | Recommended |\n|----------|---------|-------------|\n| macOS | 13.0 (Ventura) | 14.x+ |\n| Linux | Ubuntu 22.04 / RHEL 9 | Ubuntu 24.04 |\n| Windows | Windows 10 22H2 | Windows 11 |\n| CPU | x86-64 or ARM64 | ARM64 (Apple Silicon) |\n| RAM | 4 GB | 16 GB+ |\n| Disk | 2 GB free | SSD recommended |\n\n## macOS\n\n### Homebrew (Recommended)\n\n```bash\nbrew tap neuraldb/tap\nbrew install neuraldb\n\n# Start as a service (auto-restart on login)\nbrew services start neuraldb\n\n# Or start manually (foreground)\nneuraldb start\n\n# Check status\nneuraldb status\n```\n\nThe Homebrew formula installs:\n- `neuraldb` \u2014 the server binary\n- `neuraldb-cli` \u2014 an enhanced SQL shell (psql-compatible)\n- Configuration at `$(brew --prefix)/etc/neuraldb/neuraldb.conf`\n- Data directory at `$(brew --prefix)/var/neuraldb`\n\n### Direct Binary\n\n```bash\n# Apple Silicon (M1/M2/M3/M4)\ncurl -LO https://releases.neuraldb.io/1.0/neuraldb-macos-arm64.tar.gz\ntar -xzf neuraldb-macos-arm64.tar.gz\nsudo mv neuraldb /usr/local/bin/\nsudo mv neuraldb-cli /usr/local/bin/\n\n# Intel Mac\ncurl -LO https://releases.neuraldb.io/1.0/neuraldb-macos-amd64.tar.gz\ntar -xzf neuraldb-macos-amd64.tar.gz\nsudo mv neuraldb /usr/local/bin/\nsudo mv neuraldb-cli /usr/local/bin/\n```\n\n## Linux\n\n### Ubuntu / Debian\n\n```bash\n# Add the NeuralDB APT repository\ncurl -fsSL https://packages.neuraldb.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/neuraldb-keyring.gpg\necho \"deb [signed-by=/usr/share/keyrings/neuraldb-keyring.gpg] https://packages.neuraldb.io/apt stable main\" \\\n | sudo tee /etc/apt/sources.list.d/neuraldb.list\n\nsudo apt update\nsudo apt install -y neuraldb\n\n# Start and enable the service\nsudo systemctl enable --now neuraldb\n```\n\n### RHEL / Fedora / CentOS\n\n```bash\n# Add the NeuralDB YUM repository\nsudo rpm --import https://packages.neuraldb.io/gpg\nsudo tee /etc/yum.repos.d/neuraldb.repo <<'EOF'\n[neuraldb]\nname=NeuralDB Repository\nbaseurl=https://packages.neuraldb.io/rpm/stable\nenabled=1\ngpgcheck=1\ngpgkey=https://packages.neuraldb.io/gpg\nEOF\n\nsudo dnf install -y neuraldb\nsudo systemctl enable --now neuraldb\n```\n\n### Direct Binary (Linux)\n\n```bash\n# x86-64\ncurl -LO https://releases.neuraldb.io/1.0/neuraldb-linux-amd64.tar.gz\ntar -xzf neuraldb-linux-amd64.tar.gz\nsudo mv neuraldb neuraldb-cli /usr/local/bin/\n```\n\n## Windows\n\n### winget\n\n```powershell\nwinget install NeuralDB.NeuralDB\n```\n\nThis installs NeuralDB and registers it as a Windows Service. It starts automatically after installation.\n\n### Chocolatey\n\n```powershell\nchoco install neuraldb\n```\n\n### MSI Installer\n\nDownload the MSI installer from [neuraldb.io/download](https://neuraldb.io/download) and run it. The installer:\n1. Installs `neuraldb.exe` and `neuraldb-cli.exe` to `C:\\Program Files\\NeuralDB\\`\n2. Adds them to `PATH`\n3. Creates a `NeuralDB` Windows Service\n4. Initialises the data directory at `%APPDATA%\\NeuralDB\\data`\n\n## First-Time Setup\n\nAfter installation, initialise the database:\n\n```bash\n# Linux / macOS\nneuraldb init\nneuraldb start\n\n# Connect\nneuraldb-cli\n# psql prompt: neuraldb=#\n```\n\n### Change the Default Password\n\n```sql\nALTER USER neuraldb PASSWORD 'your-new-password';\n```\n\n### Create a Development Database\n\n```sql\nCREATE DATABASE myapp;\n\\c myapp\n\nCREATE TABLE documents (\n id UUID PRIMARY KEY DEFAULT gen_random_uuid(),\n content TEXT,\n embedding VECTOR(1536)\n);\n\nCREATE INDEX ON documents USING hnsw (embedding vector_cosine_ops);\n```\n\n## Configuration\n\nThe default configuration file locations:\n\n| Platform | Path |\n|----------|------|\n| macOS (Homebrew) | `$(brew --prefix)/etc/neuraldb/neuraldb.conf` |\n| Linux | `/etc/neuraldb/neuraldb.conf` |\n| Windows | `%PROGRAMDATA%\\NeuralDB\\neuraldb.conf` |\n\nFor local development, create a `neuraldb.conf` in your project directory and point to it:\n\n```bash\nneuraldb start --config ./neuraldb.conf\n```\n\nUseful development settings:\n\n```ini\n# neuraldb.conf (development)\nlisten_addresses = 'localhost'\nport = 5432\nmax_connections = 50\nshared_buffers = 256MB\nvector_buffer = 1GB\nlog_min_duration_statement = 100 # log slow queries (>100ms)\nlog_statement = 'all' # log all SQL (useful for debugging)\n```\n\n## Uninstalling\n\n```bash\n# macOS\nbrew services stop neuraldb\nbrew uninstall neuraldb\n\n# Ubuntu\nsudo systemctl stop neuraldb\nsudo apt remove neuraldb\n\n# Windows\nwinget uninstall NeuralDB.NeuralDB\n# Or: Settings \u2192 Apps \u2192 Installed Apps \u2192 NeuralDB \u2192 Uninstall\n```" + }, + { + "file": "pages/nql-aggregations.md", + "title": "Aggregations", + "section-id": "query-language", + "keywords": "aggregations, GROUP BY, COUNT, SUM, vectors, AVG, centroid, analytics", + "description": "Aggregating data in NQL including GROUP BY, COUNT, SUM, and vector-specific aggregation functions", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Aggregations\n\nNQL supports the full SQL aggregation toolkit, extended with vector-specific aggregate functions for centroid computation, clustering, and semantic analytics.\n\n## Standard Aggregations\n\nAll standard SQL aggregate functions work as expected:\n\n```sql\n-- Count documents by category\nSELECT category, COUNT(*) AS doc_count\nFROM documents\nGROUP BY category\nORDER BY doc_count DESC;\n\n-- Average price by category\nSELECT category,\n COUNT(*) AS products,\n AVG(price) AS avg_price,\n MIN(price) AS min_price,\n MAX(price) AS max_price,\n SUM(stock * price) AS inventory_value\nFROM products\nWHERE available = true\nGROUP BY category\nORDER BY inventory_value DESC;\n```\n\n## Vector Aggregations\n\n### `AVG(embedding)` \u2014 Centroid Computation\n\nCompute the centroid (average vector) of a group:\n\n```sql\n-- Centroid of all \"technology\" documents\nSELECT AVG(embedding) AS centroid\nFROM documents\nWHERE category = 'technology';\n```\n\nUse centroids to find documents representative of a cluster:\n\n```sql\nWITH centroid AS (\n SELECT AVG(embedding) AS c FROM documents WHERE category = 'technology'\n)\nSELECT id, title, 1 - (embedding <=> centroid.c) AS similarity_to_centroid\nFROM documents, centroid\nWHERE category = 'technology'\nORDER BY embedding <=> centroid.c\nLIMIT 10;\n```\n\n### `vector_centroid(embedding)` \u2014 Weighted Centroid\n\nCompute a weighted centroid using a score column:\n\n```sql\n-- Weighted centroid by rating (higher-rated items pull more)\nSELECT vector_centroid(embedding, rating) AS weighted_centroid\nFROM products\nWHERE category = 'electronics';\n```\n\n### `vector_agg_concat(embedding)` \u2014 Vector Array\n\nCollect vectors into an array for downstream processing:\n\n```sql\nSELECT category, vector_agg_concat(embedding) AS all_embeddings\nFROM documents\nGROUP BY category;\n```\n\n## GROUP BY with Vector Search\n\nFind the best document in each category for a given query:\n\n```sql\nSELECT DISTINCT ON (category)\n id, category, title, 1 - (embedding <=> $1) AS similarity\nFROM documents\nWHERE embedding IS NOT NULL\nORDER BY category, embedding <=> $1;\n```\n\nOr using a lateral join for more control:\n\n```sql\nSELECT cat.category, top_doc.id, top_doc.title, top_doc.similarity\nFROM (SELECT DISTINCT category FROM documents) cat,\nLATERAL (\n SELECT id, title, 1 - (embedding <=> $1) AS similarity\n FROM documents\n WHERE category = cat.category\n ORDER BY embedding <=> $1\n LIMIT 1\n) top_doc;\n```\n\n## Window Functions\n\nUse window functions to rank results within partitions:\n\n```sql\n-- Rank documents by similarity within each category\nSELECT\n id, title, category,\n 1 - (embedding <=> $1) AS similarity,\n RANK() OVER (\n PARTITION BY category\n ORDER BY embedding <=> $1\n ) AS rank_in_category\nFROM documents\nWHERE 1 - (embedding <=> $1) > 0.5\nORDER BY category, rank_in_category;\n```\n\nRolling average similarity over time:\n\n```sql\nSELECT\n date_trunc('day', created_at) AS day,\n AVG(1 - (embedding <=> $1)) AS avg_daily_similarity,\n AVG(AVG(1 - (embedding <=> $1))) OVER (\n ORDER BY date_trunc('day', created_at)\n ROWS BETWEEN 6 PRECEDING AND CURRENT ROW\n ) AS rolling_7d_avg\nFROM documents\nGROUP BY day\nORDER BY day;\n```\n\n## Clustering with GROUP BY\n\nPerform k-means style clustering by assigning documents to their nearest centroid:\n\n```sql\n-- Given pre-computed centroids in a centroids table:\nSELECT d.id, d.content,\n c.cluster_id,\n (d.embedding <=> c.centroid) AS distance_to_centroid\nFROM documents d\nCROSS JOIN LATERAL (\n SELECT cluster_id, centroid\n FROM centroids\n ORDER BY d.embedding <=> centroid\n LIMIT 1\n) c;\n```\n\n## HAVING with Vector Conditions\n\n```sql\n-- Categories where the average intra-category similarity is high (tight clusters)\nSELECT category,\n COUNT(*) AS doc_count,\n 1 - AVG(embedding <=> (SELECT AVG(e2.embedding) FROM documents e2 WHERE e2.category = e.category)) AS cohesion\nFROM documents e\nGROUP BY category\nHAVING COUNT(*) > 10\nORDER BY cohesion DESC;\n```\n\n## Time-Series Analytics\n\nAnalyse how semantic content shifts over time:\n\n```sql\n-- Daily semantic drift: how different is today's content from last week's?\nWITH weekly_centroids AS (\n SELECT\n date_trunc('week', created_at) AS week,\n AVG(embedding) AS centroid\n FROM documents\n GROUP BY week\n)\nSELECT\n w1.week,\n 1 - (w1.centroid <=> w2.centroid) AS similarity_to_prev_week\nFROM weekly_centroids w1\nLEFT JOIN weekly_centroids w2\n ON w2.week = w1.week - INTERVAL '1 week'\nORDER BY w1.week;\n```\n\n## JSON Aggregation with Vectors\n\nCombine JSON aggregation with vector results:\n\n```sql\nSELECT\n category,\n COUNT(*) AS total,\n AVG(price) AS avg_price,\n JSON_AGG(\n JSON_BUILD_OBJECT('id', id, 'name', name, 'similarity', 1 - (embedding <=> $1))\n ORDER BY embedding <=> $1\n ) FILTER (WHERE ROW_NUMBER() OVER (PARTITION BY category ORDER BY embedding <=> $1) <= 3)\n AS top_3_per_category\nFROM products\nWHERE available = true\nGROUP BY category;\n```\n\n## ROLLUP and CUBE\n\nStandard SQL ROLLUP and CUBE work for hierarchical aggregati" + }, + { + "file": "pages/nql-basics.md", + "title": "NQL Basics", + "section-id": "query-language", + "keywords": "NQL, NeuralDB Query Language, SQL, syntax, basics, queries", + "description": "Introduction to NeuralDB Query Language (NQL) \u2014 syntax, data types, and basic operations", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# NQL Basics\n\nNQL (NeuralDB Query Language) is a superset of standard SQL. Every valid SQL statement is also valid NQL. NQL adds extensions for vector operations, embedding generation, and semantic search primitives.\n\nIf you know SQL, you already know most of NQL. This page covers the NQL-specific additions and the data types introduced for AI workloads.\n\n## Connecting\n\nNeuralDB speaks the PostgreSQL wire protocol. Connect with any PostgreSQL client:\n\n```bash\n# psql\npsql -h localhost -p 5432 -U neuraldb -d mydb\n\n# neuraldb-cli (enhanced interactive shell)\nneuraldb-cli -h localhost\n```\n\nConnection string format:\n\n```\npostgresql://[user[:password]@][host][:port][/dbname][?param=value...]\n```\n\n## Data Types\n\n### VECTOR(n)\n\nThe core NQL extension. Stores a fixed-length array of 32-bit floats representing a vector embedding:\n\n```sql\n-- Declare a vector column with 1536 dimensions (OpenAI ada-002 output)\nCREATE TABLE documents (\n id UUID PRIMARY KEY DEFAULT gen_random_uuid(),\n content TEXT NOT NULL,\n embedding VECTOR(1536)\n);\n```\n\nInsert a vector by providing a bracketed float array:\n\n```sql\nINSERT INTO documents (content, embedding)\nVALUES ('Hello, world', '[0.1, 0.2, 0.3, ...]'); -- 1536 values\n```\n\n### HALFVEC(n)\n\nA 16-bit float variant of VECTOR. Half the memory, slight precision loss. Useful when vector_buffer is a constraint:\n\n```sql\nembedding HALFVEC(1536)\n```\n\n### SPARSEVEC(n)\n\nSparse vector representation \u2014 stores only non-zero elements. Efficient for high-dimensional but sparse vectors (e.g., BM25 term-frequency vectors):\n\n```sql\nbm25_vector SPARSEVEC(30000)\n```\n\n## Basic CRUD\n\n### Creating Tables\n\n```sql\nCREATE TABLE products (\n id UUID PRIMARY KEY DEFAULT gen_random_uuid(),\n name TEXT NOT NULL,\n description TEXT,\n category TEXT,\n price DECIMAL(10, 2),\n stock INTEGER DEFAULT 0,\n embedding VECTOR(1536),\n created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()\n);\n```\n\n### Inserting Data\n\n```sql\nINSERT INTO products (name, description, category, price, stock, embedding)\nVALUES (\n 'Wireless Headphones',\n 'Premium noise-cancelling wireless headphones with 30-hour battery',\n 'electronics',\n 299.99,\n 150,\n '[0.023, -0.187, 0.412, ...]' -- 1536 floats from your embedding model\n);\n```\n\n### Reading Data\n\nStandard SQL SELECT works as expected:\n\n```sql\nSELECT id, name, price FROM products WHERE category = 'electronics';\nSELECT * FROM products WHERE price BETWEEN 50 AND 300 AND stock > 0;\n```\n\n### Updating Data\n\n```sql\nUPDATE products SET price = 279.99, embedding = '[...]' WHERE id = $1;\n```\n\nWhen updating the `embedding` column, the HNSW index is updated atomically.\n\n### Deleting Data\n\n```sql\nDELETE FROM products WHERE id = $1;\n```\n\n## Creating Vector Indexes\n\nWithout an index, vector similarity queries perform exact linear scans (O(n)). Create an HNSW index for sub-linear performance:\n\n```sql\n-- Cosine similarity (most common for text embeddings)\nCREATE INDEX ON documents USING hnsw (embedding vector_cosine_ops);\n\n-- Euclidean distance\nCREATE INDEX ON documents USING hnsw (embedding vector_l2_ops);\n\n-- Dot product (for recommendation systems)\nCREATE INDEX ON documents USING hnsw (embedding vector_ip_ops);\n```\n\nBuild an index on an existing large table in parallel:\n\n```sql\nSET max_parallel_maintenance_workers = 8;\nCREATE INDEX CONCURRENTLY ON documents USING hnsw (embedding vector_cosine_ops);\n```\n\n## Basic Vector Queries\n\n### Find Similar Documents\n\n```sql\nSELECT id, content, 1 - (embedding <=> $1) AS similarity\nFROM documents\nORDER BY embedding <=> $1\nLIMIT 10;\n```\n\nThe `<=>` operator is cosine distance (lower = more similar). Subtract from 1 for a similarity score (higher = more similar).\n\n### Distance Operators\n\n| Operator | Distance metric | Index ops |\n|----------|----------------|-----------|\n| `<=>` | Cosine distance | `vector_cosine_ops` |\n| `<->` | Euclidean (L2) distance | `vector_l2_ops` |\n| `<#>` | Negative dot product | `vector_ip_ops` |\n\n### Distance Threshold\n\n```sql\n-- Only return results with cosine similarity > 0.8\nSELECT id, content, 1 - (embedding <=> $1) AS similarity\nFROM documents\nWHERE 1 - (embedding <=> $1) > 0.8\nORDER BY embedding <=> $1\nLIMIT 20;\n```\n\n**Note:** The `WHERE 1 - (embedding <=> $1) > 0.8` condition is evaluated after the ANN search, not before. Use `LIMIT` generously enough to capture all relevant results before the threshold filter.\n\n## NQL Functions\n\n### `to_vector(text)`\n\nConvert a string literal to a VECTOR:\n\n```sql\nSELECT to_vector('[0.1, 0.2, 0.3]')::VECTOR(3);\n```\n\n### `vector_dims(v)`\n\nReturn the number of dimensions:\n\n```sql\nSELECT vector_dims(embedding) FROM documents LIMIT 1;\n-- Returns: 1536\n```\n\n### `vector_norm(v)`\n\nReturn the L2 norm of a vector:\n\n```sql\nSELECT vector_norm(embedding) FROM documents LIMIT 5;\n```\n\n### `cosine_similarity(a, b)`, `l2_distance(a, b)`, `dot_product(a, b)`\n\nNamed function alternatives to the operators:\n\n```sql\nSELECT cosine_similarity(embedding, $1) AS similarity\nFROM documents\nORDER BY similarity DESC\nLIMIT 10;\n```\n\n#" + }, + { + "file": "pages/nql-hybrid.md", + "title": "Hybrid Queries", + "section-id": "query-language", + "keywords": "hybrid queries, vector, relational, filters, combined, semantic search, metadata", + "description": "Combining vector similarity and relational filters in NQL hybrid queries", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Hybrid Queries\n\nHybrid queries combine vector similarity search with relational filter predicates in a single SQL statement. The NeuralDB query planner handles the execution strategy \u2014 you write normal SQL with vector operators.\n\n## Basic Hybrid Query\n\nFind the 10 most semantically similar products that are in the \"electronics\" category and in stock:\n\n```sql\nSELECT id, name, price, 1 - (embedding <=> $1) AS similarity\nFROM products\nWHERE category = 'electronics'\n AND stock > 0\n AND price < 500\nORDER BY embedding <=> $1\nLIMIT 10;\n```\n\nNeuralDB automatically determines whether to:\n1. **Pre-filter**: apply relational conditions first, then search the filtered set\n2. **Post-filter**: run ANN search, then apply conditions to the top-k results\n\nThe decision is based on the selectivity of the relational predicates.\n\n## Query Planner Hints\n\nOverride the planner's strategy:\n\n```sql\n-- Force pre-filter (good when relational filter is very selective)\nSELECT /*+ PREFILTER */ id, name, score\nFROM (\n SELECT id, name, 1 - (embedding <=> $1) AS score\n FROM products\n WHERE category = 'electronics' -- very selective: 2% of rows\n) sub\nORDER BY score DESC\nLIMIT 10;\n\n-- Force post-filter (good when relational filter is weakly selective)\nSELECT /*+ POSTFILTER */ id, name, 1 - (embedding <=> $1) AS score\nFROM products\nWHERE price < 500 -- weakly selective: 80% of rows\nORDER BY embedding <=> $1\nLIMIT 10;\n```\n\n## Filtering by Multiple Conditions\n\n```sql\nSELECT id, name, description,\n 1 - (embedding <=> $1) AS similarity,\n price,\n rating\nFROM products\nWHERE category = ANY($2) -- multi-category filter\n AND price BETWEEN $3 AND $4\n AND rating >= 4.0\n AND discontinued = false\n AND created_at > NOW() - INTERVAL '1 year'\nORDER BY embedding <=> $1\nLIMIT 20;\n-- $1 = query embedding\n-- $2 = ['electronics', 'computers']\n-- $3 = 50, $4 = 1000\n```\n\n## Hybrid Full-Text + Vector (BM25)\n\nCombine traditional full-text search with vector similarity using Reciprocal Rank Fusion (RRF):\n\n```sql\nWITH vector_search AS (\n SELECT id, ROW_NUMBER() OVER (ORDER BY embedding <=> $1) AS rank\n FROM documents\n ORDER BY embedding <=> $1\n LIMIT 100\n),\nfts_search AS (\n SELECT id, ROW_NUMBER() OVER (ORDER BY ts_rank_cd(tsv, query) DESC) AS rank\n FROM documents, to_tsquery('english', $2) query\n WHERE tsv @@ query\n ORDER BY ts_rank_cd(tsv, query) DESC\n LIMIT 100\n),\nrrf AS (\n SELECT\n COALESCE(v.id, f.id) AS id,\n (COALESCE(1.0 / (60 + v.rank), 0) + COALESCE(1.0 / (60 + f.rank), 0)) AS rrf_score\n FROM vector_search v\n FULL OUTER JOIN fts_search f ON v.id = f.id\n)\nSELECT d.id, d.content, rrf.rrf_score\nFROM rrf\nJOIN documents d ON d.id = rrf.id\nORDER BY rrf_score DESC\nLIMIT 10;\n```\n\nNQL also provides a built-in `HYBRID_SEARCH` function:\n\n```sql\nSELECT id, content, score\nFROM HYBRID_SEARCH(\n table := 'documents',\n vector_column := 'embedding',\n tsv_column := 'tsv',\n query_vector := $1,\n query_text := $2,\n top_k := 10,\n rrf_k := 60,\n vector_weight := 0.6,\n text_weight := 0.4\n);\n```\n\n## Joining Vector Results with Other Tables\n\n```sql\nSELECT\n p.id,\n p.name,\n p.price,\n c.name AS category_name,\n u.display_name AS seller,\n 1 - (p.embedding <=> $1) AS similarity\nFROM products p\nJOIN categories c ON c.id = p.category_id\nJOIN users u ON u.id = p.seller_id\nWHERE p.available = true\n AND c.slug = ANY($2)\n AND u.verified = true\nORDER BY p.embedding <=> $1\nLIMIT 15;\n```\n\n## Subquery Vectors\n\nUse a subquery to dynamically compute a query vector from existing data:\n\n```sql\n-- Find products similar to product #123\nSELECT id, name, 1 - (embedding <=> ref.embedding) AS similarity\nFROM products,\n (SELECT embedding FROM products WHERE id = $1) ref\nWHERE id != $1\nORDER BY embedding <=> ref.embedding\nLIMIT 10;\n```\n\n## Tenant-Scoped Search\n\nFor multi-tenant applications, always include tenant filters:\n\n```sql\nSELECT id, content, 1 - (embedding <=> $1) AS similarity\nFROM documents\nWHERE tenant_id = $2 -- partition pruning if SHARD BY tenant_id\n AND embedding IS NOT NULL\nORDER BY embedding <=> $1\nLIMIT 10;\n```\n\nIf the table is sharded by `tenant_id`, this query runs entirely on the correct shard without cross-shard coordination.\n\n## Composite Scoring\n\nCombine vector similarity with relational signals:\n\n```sql\nSELECT\n id,\n name,\n price,\n rating,\n -- Weighted composite score: 70% semantic, 20% rating, 10% recency\n (0.7 * (1 - (embedding <=> $1))\n + 0.2 * (rating / 5.0)\n + 0.1 * (1 - EXTRACT(DAYS FROM NOW() - created_at) / 365.0)\n ) AS composite_score\nFROM products\nWHERE available = true\n AND price < $2\nORDER BY composite_score DESC\nLIMIT 20;\n```\n\n## Pagination\n\nCursor-based pagination for vector results:\n\n```sql\n-- Page 1\nSELECT id, name, (embedding <=> $1) AS dist\nFROM products\nWHERE available = true\nORDER BY dist, id -- secondary sort by id for stable pagination\nLIMIT 20;\n\n-- Page 2 (cursor: last dist and id from page 1)\nSELECT id, name, (embedding <=> $1) AS dist\nFROM products\nWHERE available " + }, + { + "file": "pages/nql-transactions.md", + "title": "Transactions", + "section-id": "query-language", + "keywords": "transactions, ACID, isolation levels, MVCC, BEGIN, COMMIT, ROLLBACK", + "description": "ACID transactions in NeuralDB \u2014 isolation levels, MVCC, savepoints, and advisory locks", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Transactions\n\nNeuralDB provides full ACID transactions with MVCC (Multi-Version Concurrency Control). Unlike most vector databases, NeuralDB guarantees atomicity across both relational and vector data within a single transaction.\n\n## ACID Guarantees\n\n| Property | Guarantee |\n|----------|---------|\n| **Atomicity** | All operations in a transaction succeed or all are rolled back \u2014 including vector index updates |\n| **Consistency** | Constraints (foreign keys, unique indexes, not null) are enforced at commit time |\n| **Isolation** | Concurrent transactions do not see each other's uncommitted changes |\n| **Durability** | Committed transactions survive crashes via the WAL |\n\n## Basic Transaction Syntax\n\n```sql\nBEGIN;\n\n-- Your operations here\nINSERT INTO documents (content, embedding) VALUES ($1, $2);\nUPDATE document_stats SET total_count = total_count + 1;\nINSERT INTO audit_log (action, data) VALUES ('insert', $3);\n\nCOMMIT;\n```\n\nOn error, roll back:\n\n```sql\nBEGIN;\n\nINSERT INTO documents (content, embedding) VALUES ($1, $2);\n\n-- Something went wrong\nROLLBACK;\n```\n\n## Isolation Levels\n\nNeuralDB supports four isolation levels. Set them with `SET TRANSACTION ISOLATION LEVEL`:\n\n```sql\nBEGIN;\nSET TRANSACTION ISOLATION LEVEL READ COMMITTED;\n-- ... your queries ...\nCOMMIT;\n```\n\n### Read Committed (Default)\n\nEach statement sees only rows committed before that statement began. Two successive reads within the same transaction may see different data if another transaction commits between them.\n\n```sql\nBEGIN;\n-- Sees all rows committed before this SELECT\nSELECT COUNT(*) FROM documents; -- Returns 1000\n\n-- Another transaction inserts and commits a row here\n\n-- Sees the new row (non-repeatable read)\nSELECT COUNT(*) FROM documents; -- Returns 1001\nCOMMIT;\n```\n\n### Repeatable Read\n\nA transaction sees only rows committed before the transaction began. Reads are stable throughout the transaction.\n\n```sql\nBEGIN ISOLATION LEVEL REPEATABLE READ;\nSELECT COUNT(*) FROM documents; -- Returns 1000\n\n-- Another transaction inserts and commits\n\nSELECT COUNT(*) FROM documents; -- Still 1000 \u2014 repeatable read\nCOMMIT;\n```\n\n### Serializable\n\nThe strictest level. Transactions execute as if they ran serially one after another. NeuralDB uses Serializable Snapshot Isolation (SSI) \u2014 it allows concurrent execution but detects and aborts transactions that would produce a non-serializable outcome.\n\n```sql\nBEGIN ISOLATION LEVEL SERIALIZABLE;\n-- ... complex read-modify-write patterns ...\nCOMMIT;\n-- May raise: ERROR: could not serialize access \u2014 retry the transaction\n```\n\n### Read Uncommitted\n\nNeuralDB maps this to Read Committed (it does not implement dirty reads).\n\n## Retry Logic\n\nSerializable transactions can fail with serialization errors. Always retry:\n\n```python\nfrom psycopg2 import errors\n\nMAX_RETRIES = 5\n\nfor attempt in range(MAX_RETRIES):\n try:\n with conn.cursor() as cur:\n cur.execute(\"BEGIN ISOLATION LEVEL SERIALIZABLE\")\n # ... your operations ...\n cur.execute(\"COMMIT\")\n break\n except errors.SerializationFailure:\n conn.rollback()\n if attempt == MAX_RETRIES - 1:\n raise\n time.sleep(0.1 * (2 ** attempt)) # exponential backoff\n```\n\n## Savepoints\n\nSavepoints allow partial rollbacks within a transaction:\n\n```sql\nBEGIN;\n\nINSERT INTO documents (content, embedding) VALUES ($1, $2);\n\nSAVEPOINT after_insert;\n\n-- Risky operation\nUPDATE document_stats SET count = count + 1 WHERE id = $3;\n\n-- Oh no, something went wrong \u2014 roll back to the savepoint\nROLLBACK TO SAVEPOINT after_insert;\n\n-- The INSERT is still pending \u2014 we can try a different approach\nUPDATE document_stats SET count = count + 1 WHERE id = $4;\n\nCOMMIT;\n```\n\n## Vector Transactions\n\nVector index updates are transactional in NeuralDB. An HNSW index entry is added atomically with the row:\n\n```sql\nBEGIN;\n\n-- Both the row and the vector index entry are inserted atomically\nINSERT INTO documents (id, content, embedding) VALUES ($1, $2, $3);\n\n-- If we ROLLBACK, neither the row nor the index entry will exist\nROLLBACK;\n\n-- After rollback, a similarity search will NOT find $1\nSELECT id FROM documents ORDER BY embedding <=> $3 LIMIT 1;\n-- $1 is not returned\n```\n\n## Long-Running Transactions\n\nAvoid long-running transactions \u2014 they:\n- Hold row-level locks, blocking other writes\n- Prevent VACUUM from reclaiming dead rows (bloat)\n- Increase the risk of deadlocks\n\nSet a statement timeout to kill runaway queries:\n\n```sql\nSET statement_timeout = '30s';\n```\n\nSet a transaction timeout:\n\n```sql\nSET idle_in_transaction_session_timeout = '5min';\n```\n\n## Deadlock Detection\n\nNeuralDB automatically detects deadlocks and aborts one of the transactions:\n\n```\nERROR: deadlock detected\nDETAIL: Process 12345 waits for ShareLock on transaction 67890; blocked by process 99999.\nHint: See server log for query details.\n```\n\nMinimise deadlock risk by always acquiring locks in the same order across all transactions.\n\n## Advisory Locks\n\nFor applicat" + }, + { + "file": "pages/nql-vectors.md", + "title": "Vector Queries", + "section-id": "query-language", + "keywords": "vector queries, NEAREST, SIMILAR, cosine, dot product, euclidean, ANN", + "description": "Writing vector similarity queries in NQL \u2014 NEAREST, SIMILAR, distance operators, and recall tuning", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Vector Queries\n\nNQL extends standard SQL with operators and functions for vector similarity search. This page covers every method for querying vectors, from basic nearest-neighbour lookups to advanced recall tuning.\n\n## Distance Operators\n\nNQL provides three distance operators that double as index-acceleration hints:\n\n```sql\n-- Cosine distance (returns 0 to 2, lower = more similar)\nembedding <=> query_vector\n\n-- Euclidean (L2) distance (returns 0 to \u221e, lower = more similar)\nembedding <-> query_vector\n\n-- Negative dot product (higher inner product = more similar \u2192 negate for ORDER BY)\nembedding <#> query_vector\n```\n\nAlways pair `ORDER BY` with `LIMIT` when using distance operators \u2014 the planner uses the HNSW index only when there is an explicit `ORDER BY ... LIMIT`:\n\n```sql\n-- \u2705 Uses HNSW index\nSELECT id, content FROM documents\nORDER BY embedding <=> '[0.1, 0.2, ...]'\nLIMIT 10;\n\n-- \u274c Full scan (no ORDER BY ... LIMIT)\nSELECT id, content FROM documents\nWHERE (embedding <=> '[0.1, 0.2, ...]') < 0.3;\n```\n\n## NEAREST Clause\n\nNQL provides a syntactic alternative to `ORDER BY ... LIMIT` for nearest-neighbour queries:\n\n```sql\nSELECT id, content, score\nFROM documents\nNEAREST TO embedding = '[0.1, 0.2, ...]' USING COSINE\nTOP 10;\n```\n\nThis is equivalent to:\n\n```sql\nSELECT id, content, 1 - (embedding <=> '[0.1, 0.2, ...]') AS score\nFROM documents\nORDER BY embedding <=> '[0.1, 0.2, ...]'\nLIMIT 10;\n```\n\nThe `NEAREST TO` clause is more readable and allows NeuralDB to apply additional optimisations.\n\n### Distance Metrics in NEAREST\n\n```sql\nNEAREST TO embedding = $1 USING COSINE TOP 10\nNEAREST TO embedding = $1 USING EUCLIDEAN TOP 10\nNEAREST TO embedding = $1 USING DOT_PRODUCT TOP 10\n```\n\n## SIMILAR Clause\n\n`SIMILAR` returns results above a similarity threshold rather than a fixed count. Because the threshold is checked after the ANN search, NeuralDB must retrieve an initial candidate set. Use `LIMIT` to cap the candidates:\n\n```sql\nSELECT id, content, score\nFROM documents\nSIMILAR TO embedding = $1 USING COSINE THRESHOLD 0.75\nLIMIT 100;\n```\n\nThis returns up to 100 documents with cosine similarity \u2265 0.75.\n\n## Returning Scores\n\nInclude the distance or similarity score in results:\n\n```sql\n-- Distance (lower = more similar)\nSELECT id, content, (embedding <=> $1) AS distance\nFROM documents\nORDER BY embedding <=> $1\nLIMIT 10;\n\n-- Similarity (higher = more similar, cosine)\nSELECT id, content, 1 - (embedding <=> $1) AS similarity\nFROM documents\nORDER BY embedding <=> $1\nLIMIT 10;\n```\n\n## Querying With a Vector Literal\n\nPass vectors as SQL parameters (recommended) or literals:\n\n```sql\n-- Parameterised (prevents injection, preferred)\nSELECT id, content FROM documents\nORDER BY embedding <=> $1\nLIMIT 10;\n-- $1 = '[0.023, -0.187, 0.412, ...]'\n\n-- Inline literal (useful in SQL shells)\nSELECT id, content FROM documents\nORDER BY embedding <=> '[0.023, -0.187, 0.412]'::VECTOR(3)\nLIMIT 5;\n```\n\n## Recall Tuning\n\nThe HNSW index trades recall for performance. By default, `hnsw.ef_search = 40`, which provides ~95% recall at ~1ms latency for 10M vectors.\n\nIncrease `ef_search` for higher recall:\n\n```sql\n-- Set for the current session\nSET hnsw.ef_search = 200;\n\n-- Set for the current transaction\nBEGIN;\nSET LOCAL hnsw.ef_search = 200;\nSELECT * FROM documents ORDER BY embedding <=> $1 LIMIT 10;\nCOMMIT;\n```\n\nTypical recall vs performance trade-off (10M 1536-dim vectors, 32 vCPU):\n\n| ef_search | Recall@10 | p50 latency | QPS |\n|-----------|-----------|-------------|-----|\n| 20 | 89% | 0.7ms | 12,000 |\n| 40 | 95% | 1.2ms | 8,400 |\n| 80 | 98% | 2.1ms | 4,800 |\n| 200 | 99.5% | 4.8ms | 2,100 |\n| exact | 100% | 45ms | 220 |\n\n## Exact Search\n\nForce exact (brute-force) nearest-neighbour search, ignoring the HNSW index:\n\n```sql\nSET neuraldb.vector_scan = 'exact';\nSELECT * FROM documents ORDER BY embedding <=> $1 LIMIT 10;\nRESET neuraldb.vector_scan;\n```\n\nUse exact search when:\n- You need 100% recall (e.g., de-duplication, exact compliance checks)\n- The table has fewer than ~100k rows (exact is competitive)\n- You are benchmarking ANN recall\n\n## Bulk Vector Operations\n\n### Batch Insert\n\nUse `COPY` for high-throughput ingestion:\n\n```bash\n# Format: id\\tcontent\\tembedding\npsql -c \"\\COPY documents (id, content, embedding) FROM '/data/vectors.tsv'\"\n```\n\n### Updating Embeddings in Bulk\n\n```sql\nUPDATE documents\nSET embedding = new_embeddings.embedding\nFROM (VALUES\n ('uuid-1', '[...]'::VECTOR(1536)),\n ('uuid-2', '[...]'::VECTOR(1536))\n) AS new_embeddings(id, embedding)\nWHERE documents.id = new_embeddings.id::UUID;\n```\n\n## Multi-Vector Queries\n\nFind documents closest to ANY of multiple query vectors (OR semantics):\n\n```sql\nWITH queries AS (\n SELECT UNNEST(ARRAY['[...]'::VECTOR(1536), '[...]'::VECTOR(1536)]) AS qv\n),\nranked AS (\n SELECT d.id, d.content, MIN(d.embedding <=> q.qv) AS best_distance\n FROM documents d, queries q\n GROUP BY d.id, d.content\n)\nSELECT * FROM ranked\nORDER BY best_distance\nLIMIT 20;\n```\n\n## Vector Arithmetic\n\nNQL supports vector arithmetic for " + }, + { + "file": "pages/ops-backup.md", + "title": "Backup & Restore", + "section-id": "operations", + "keywords": "backup, restore, snapshot, WAL archiving, PITR, point-in-time recovery", + "description": "Backup and restore strategies for NeuralDB \u2014 snapshots, WAL archiving, and point-in-time recovery", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Backup & Restore\n\nA comprehensive backup strategy for NeuralDB combines base snapshots with continuous WAL archiving, enabling point-in-time recovery (PITR) to any moment within your retention window.\n\n## Backup Strategies\n\n| Strategy | Recovery point objective | Recovery time | Storage |\n|----------|--------------------------|---------------|---------|\n| Snapshot only | Time of last snapshot | Fast | Medium |\n| WAL archiving only | Continuous (any point) | Slow | High |\n| Snapshot + WAL | Best of both | Fast | High |\n\n**Recommendation:** Use snapshot + WAL archiving in production. Take daily base snapshots and archive WAL continuously.\n\n## Physical Snapshot (pg_basebackup)\n\n`pg_basebackup` creates a consistent physical copy of the data directory:\n\n```bash\n# Full backup \u2014 local filesystem\npg_basebackup \\\n --host=localhost \\\n --port=5432 \\\n --username=backup_user \\\n --pgdata=/backups/neuraldb/$(date +%Y%m%d) \\\n --wal-method=stream \\\n --checkpoint=fast \\\n --compress=lz4 \\\n --progress \\\n --verbose\n\n# Full backup \u2014 tar format (smaller, easier to upload to S3)\npg_basebackup \\\n --host=localhost \\\n --pgdata=- \\\n --format=tar \\\n --wal-method=stream \\\n --compress=lz4 \\\n | aws s3 cp - s3://my-backups/neuraldb/base-$(date +%Y%m%d).tar.lz4\n```\n\nCreate a dedicated backup user:\n\n```sql\nCREATE USER backup_user WITH REPLICATION PASSWORD 'backup-password';\nGRANT CONNECT ON DATABASE neuraldb TO backup_user;\n```\n\n## WAL Archiving\n\nWAL archiving copies each WAL segment to a secure location as it is completed. Combined with a base snapshot, this enables PITR.\n\nEnable WAL archiving:\n\n```ini\n# neuraldb.conf\nwal_level = replica\narchive_mode = on\narchive_command = 'aws s3 cp %p s3://my-backups/neuraldb/wal/%f'\narchive_timeout = 60 # archive at least every 60 seconds even if no WAL activity\n```\n\nVerify archiving is working:\n\n```sql\nSELECT last_archived_wal, last_archived_time,\n last_failed_wal, last_failed_time,\n archived_count, failed_count\nFROM pg_stat_archiver;\n```\n\n### S3 Archive Command\n\n```bash\n#!/bin/bash\n# /usr/local/bin/neuraldb-archive.sh\n# Usage: %p = source file path, %f = file name\n\nset -e\nSOURCE=\"$1\"\nDEST_FILE=\"$2\"\nS3_BUCKET=\"${ARCHIVE_S3_BUCKET}\"\nS3_PREFIX=\"${ARCHIVE_S3_PREFIX:-neuraldb/wal/}\"\n\naws s3 cp \"$SOURCE\" \"s3://${S3_BUCKET}/${S3_PREFIX}${DEST_FILE}\" \\\n --storage-class STANDARD_IA \\\n --sse aws:kms\n```\n\n```ini\narchive_command = '/usr/local/bin/neuraldb-archive.sh %p %f'\n```\n\n## Automated Backups with pgBackRest\n\npgBackRest is the recommended tool for production NeuralDB backups:\n\n```bash\n# Install\nsudo apt install pgbackrest\n\n# Configure\nsudo tee /etc/pgbackrest/pgbackrest.conf <<'EOF'\n[global]\nrepo1-path=/var/lib/pgbackrest\nrepo1-retention-full=7\nrepo1-retention-diff=14\nrepo1-type=s3\nrepo1-s3-bucket=my-neuraldb-backups\nrepo1-s3-endpoint=s3.amazonaws.com\nrepo1-s3-region=us-east-1\ncompress-type=lz4\nstart-fast=y\nbackup-standby=y\n\n[neuraldb]\npg1-path=/var/lib/neuraldb/data\npg1-port=5432\npg1-user=backup_user\nEOF\n\n# Initialise\nsudo -u postgres pgbackrest --stanza=neuraldb stanza-create\n\n# Full backup\nsudo -u postgres pgbackrest --stanza=neuraldb backup --type=full\n\n# Differential backup (only changes since last full)\nsudo -u postgres pgbackrest --stanza=neuraldb backup --type=diff\n\n# Incremental (only changes since last backup of any type)\nsudo -u postgres pgbackrest --stanza=neuraldb backup --type=incr\n```\n\nSchedule backups with cron:\n\n```cron\n# /etc/cron.d/neuraldb-backup\n0 1 * * 0 postgres pgbackrest --stanza=neuraldb backup --type=full\n0 1 * * 1-6 postgres pgbackrest --stanza=neuraldb backup --type=diff\n```\n\n## Point-in-Time Recovery (PITR)\n\nTo restore to a specific point in time:\n\n```bash\n# Stop NeuralDB\nsystemctl stop neuraldb\n\n# Restore a base backup\npgbackrest --stanza=neuraldb restore \\\n --target=\"2026-05-15 14:30:00+00\" \\\n --target-action=promote \\\n --delta\n\n# Or restore to just before a specific transaction\npgbackrest --stanza=neuraldb restore \\\n --target-name=\"before_accidental_delete\" \\\n --target-action=promote\n\n# Start NeuralDB \u2014 it will replay WAL up to the target point\nsystemctl start neuraldb\n```\n\nCreate named restore points before risky operations:\n\n```sql\n-- Before running a migration\nSELECT pg_create_restore_point('before_migration_20260515');\n```\n\n## Logical Backup (pg_dump)\n\nFor smaller databases or table-level backups, `pg_dump` provides a logical backup:\n\n```bash\n# Dump entire database\npg_dump -h localhost -U neuraldb mydb | \\\n lz4 | \\\n aws s3 cp - s3://my-backups/neuraldb/logical-$(date +%Y%m%d).sql.lz4\n\n# Dump specific table\npg_dump -h localhost -U neuraldb -t documents mydb > documents-backup.sql\n\n# Dump in custom format (best compression, selective restore)\npg_dump -Fc -h localhost -U neuraldb mydb > mydb-$(date +%Y%m%d).dump\n```\n\n**Note:** Logical backups do not include vector index data \u2014 only the raw vector column values. After restore, recreate indexes manually.\n\n## Restoring from pg_dump\n\n```bash\n# Restore entire database\nlz4 -d backup.sql.lz4" + }, + { + "file": "pages/ops-migration.md", + "title": "Migration", + "section-id": "operations", + "keywords": "migration, import, Postgres, Pinecone, Weaviate, data migration, ETL", + "description": "Migrating data to NeuralDB from PostgreSQL, Pinecone, Weaviate, and other sources", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Migration\n\nThis guide covers migrating data into NeuralDB from common sources: PostgreSQL (with or without pgvector), Pinecone, and Weaviate.\n\n## From PostgreSQL (without vectors)\n\nIf you are migrating a standard PostgreSQL database to NeuralDB, the simplest path is a logical dump and restore:\n\n```bash\n# 1. Dump from source Postgres\npg_dump \\\n -h source-host \\\n -U source-user \\\n -d source-database \\\n --format=custom \\\n --compress=9 \\\n > source-backup.dump\n\n# 2. Create the target database in NeuralDB\npsql -h neuraldb-host -U neuraldb -c \"CREATE DATABASE myapp;\"\n\n# 3. Restore into NeuralDB\npg_restore \\\n -h neuraldb-host \\\n -U neuraldb \\\n -d myapp \\\n --jobs=8 \\\n --no-owner \\\n source-backup.dump\n```\n\n### Adding Vector Columns Post-Migration\n\nAfter restoring the schema and data, add vector columns and generate embeddings:\n\n```sql\n-- Add the vector column\nALTER TABLE documents ADD COLUMN embedding VECTOR(1536);\n\n-- Create the index (do this before backfilling on large tables)\nCREATE INDEX CONCURRENTLY documents_embedding_idx\nON documents USING hnsw (embedding vector_cosine_ops);\n```\n\nThen backfill embeddings in batches:\n\n```python\nimport openai\nfrom neuraldb import NeuralDB\n\nclient = NeuralDB(connection_string)\nopenai_client = openai.OpenAI()\n\nBATCH_SIZE = 100\n\nwhile True:\n rows = client.query(\"\"\"\n SELECT id, content FROM documents\n WHERE embedding IS NULL\n LIMIT %s\n \"\"\", [BATCH_SIZE])\n\n if not rows:\n break\n\n texts = [row['content'] for row in rows]\n response = openai_client.embeddings.create(\n model=\"text-embedding-3-small\",\n input=texts\n )\n\n updates = [\n (response.data[i].embedding, rows[i]['id'])\n for i in range(len(rows))\n ]\n\n client.executemany(\n \"UPDATE documents SET embedding = %s WHERE id = %s\",\n updates\n )\n print(f\"Backfilled {len(rows)} rows\")\n```\n\n## From PostgreSQL + pgvector\n\npgvector uses the same `VECTOR` type as NeuralDB. Migration is a direct dump and restore with minimal adjustments.\n\n```bash\n# Dump \u2014 exclude pgvector extension (NeuralDB has native vector support)\npg_dump \\\n -h source-host -U source-user -d source-db \\\n --format=custom \\\n --exclude-extension=vector \\\n > pgvector-backup.dump\n\npg_restore \\\n -h neuraldb-host -U neuraldb -d myapp \\\n --jobs=8 \\\n pgvector-backup.dump\n```\n\n### Re-create HNSW Indexes\n\npgvector HNSW indexes are not transferred. Recreate them in NeuralDB:\n\n```sql\n-- Drop pgvector-created indexes\nDROP INDEX IF EXISTS documents_embedding_idx;\n\n-- Create NeuralDB HNSW index (same syntax, better performance)\nCREATE INDEX CONCURRENTLY documents_embedding_idx\nON documents USING hnsw (embedding vector_cosine_ops)\nWITH (m = 16, ef_construction = 64);\n```\n\n## From Pinecone\n\nPinecone stores vectors with metadata. Export using the Pinecone SDK and ingest into NeuralDB:\n\n```python\nimport pinecone\nfrom neuraldb import NeuralDB, BulkIngestor\n\n# Source: Pinecone\npc = pinecone.Pinecone(api_key=os.environ[\"PINECONE_API_KEY\"])\nindex = pc.Index(\"my-index\")\n\n# Target: NeuralDB\nclient = NeuralDB(os.environ[\"NEURALDB_URL\"])\n\n# Create target table\nclient.execute(\"\"\"\n CREATE TABLE IF NOT EXISTS pinecone_migration (\n id TEXT PRIMARY KEY,\n embedding VECTOR(1536),\n metadata JSONB,\n migrated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()\n )\n\"\"\")\n\nclient.execute(\"\"\"\n CREATE INDEX IF NOT EXISTS pinecone_migration_emb_idx\n ON pinecone_migration USING hnsw (embedding vector_cosine_ops)\n\"\"\")\n\n# Paginate through all Pinecone vectors\ningestor = BulkIngestor(client, table=\"pinecone_migration\", batch_size=500)\n\nwith ingestor as ing:\n for ids_batch in paginate_pinecone_ids(index, batch_size=1000):\n fetch_response = index.fetch(ids=ids_batch)\n\n for vector_id, vector_data in fetch_response.vectors.items():\n ing.add({\n \"id\": vector_id,\n \"embedding\": vector_data.values,\n \"metadata\": vector_data.metadata or {}\n })\n\nprint(f\"Migrated {ingestor.total_inserted} vectors\")\n```\n\n### Mapping Pinecone Metadata to Columns\n\nFlatten commonly-queried metadata fields into dedicated columns for better query performance:\n\n```python\n# Instead of: metadata JSONB\n# Create typed columns for common filter fields:\nclient.execute(\"\"\"\n ALTER TABLE pinecone_migration\n ADD COLUMN IF NOT EXISTS category TEXT GENERATED ALWAYS AS (metadata->>'category') STORED,\n ADD COLUMN IF NOT EXISTS created_date DATE GENERATED ALWAYS AS ((metadata->>'date')::DATE) STORED;\n\n CREATE INDEX ON pinecone_migration (category);\n CREATE INDEX ON pinecone_migration (created_date);\n\"\"\")\n```\n\n## From Weaviate\n\nExport Weaviate data using the Weaviate client SDK:\n\n```python\nimport weaviate\nfrom neuraldb import NeuralDB, BulkIngestor\n\nweaviate_client = weaviate.connect_to_local()\nneuraldb_client = NeuralDB(os.environ[\"NEURALDB_URL\"])\n\ncollection = weaviate_client.collections.get(\"Document\")\n\n# Create target schema\nne" + }, + { + "file": "pages/ops-monitoring.md", + "title": "Monitoring", + "section-id": "operations", + "keywords": "monitoring, Prometheus, Grafana, metrics, alerts, observability, dashboards", + "description": "Monitoring NeuralDB with Prometheus metrics, Grafana dashboards, and alert configuration", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Monitoring\n\n![NeuralDB Dashboard](assets/images/dashboard.jpg)\n\nObservability is critical for database operations. NeuralDB exposes Prometheus-compatible metrics and provides an official Grafana dashboard for real-time monitoring.\n\n## Prometheus Metrics\n\nNeuralDB exposes metrics at `http://localhost:9187/metrics` (via the bundled exporter).\n\nEnable the metrics exporter:\n\n```ini\n# neuraldb.conf\nmetrics.enabled = true\nmetrics.port = 9187\nmetrics.path = /metrics\n```\n\nOr run the standalone exporter:\n\n```bash\nneuraldb_exporter \\\n --web.listen-address=:9187 \\\n --db.uri=\"postgresql://monitor:password@localhost:5432/neuraldb?sslmode=disable\"\n```\n\n### Key Metrics\n\n#### Connection Metrics\n\n| Metric | Type | Description |\n|--------|------|-------------|\n| `neuraldb_connections_total` | Gauge | Current connections by state |\n| `neuraldb_connections_max` | Gauge | `max_connections` setting |\n| `neuraldb_connection_pool_waiting` | Gauge | Queries waiting for a connection |\n\n#### Query Metrics\n\n| Metric | Type | Description |\n|--------|------|-------------|\n| `neuraldb_queries_total` | Counter | Total queries by database and status |\n| `neuraldb_query_duration_seconds` | Histogram | Query duration (p50, p95, p99) |\n| `neuraldb_slow_queries_total` | Counter | Queries exceeding `log_min_duration_statement` |\n| `neuraldb_deadlocks_total` | Counter | Deadlocks detected |\n\n#### Vector Metrics\n\n| Metric | Type | Description |\n|--------|------|-------------|\n| `neuraldb_vector_queries_total` | Counter | Vector similarity queries by index |\n| `neuraldb_vector_query_duration_seconds` | Histogram | ANN query latency |\n| `neuraldb_hnsw_index_size_bytes` | Gauge | In-memory size of HNSW graphs |\n| `neuraldb_hnsw_build_duration_seconds` | Histogram | Time to build HNSW indexes |\n| `neuraldb_vector_recall_ratio` | Gauge | Estimated recall for ANN queries |\n\n#### Replication Metrics\n\n| Metric | Type | Description |\n|--------|------|-------------|\n| `neuraldb_replication_lag_bytes` | Gauge | WAL lag per replica |\n| `neuraldb_replication_lag_seconds` | Gauge | Time lag per replica |\n| `neuraldb_wal_size_bytes` | Gauge | Current WAL on-disk size |\n\n#### Storage Metrics\n\n| Metric | Type | Description |\n|--------|------|-------------|\n| `neuraldb_database_size_bytes` | Gauge | Total database size |\n| `neuraldb_table_size_bytes` | Gauge | Size per table |\n| `neuraldb_bloat_ratio` | Gauge | Estimated dead row ratio |\n| `neuraldb_checkpoint_duration_seconds` | Histogram | Checkpoint write time |\n\n## Prometheus Configuration\n\n```yaml\n# prometheus.yml\nscrape_configs:\n - job_name: 'neuraldb'\n static_configs:\n - targets: ['localhost:9187']\n scrape_interval: 15s\n metrics_path: /metrics\n```\n\n## Grafana Dashboard\n\nImport the official NeuralDB dashboard from Grafana.com (Dashboard ID: **18921**):\n\n```bash\n# Import via Grafana API\ncurl -X POST \\\n http://admin:password@localhost:3000/api/dashboards/import \\\n -H \"Content-Type: application/json\" \\\n -d '{ \"gnetId\": 18921, \"overwrite\": true, \"inputs\": [{\"name\": \"DS_PROMETHEUS\", \"type\": \"datasource\", \"pluginId\": \"prometheus\", \"value\": \"Prometheus\"}] }'\n```\n\nThe dashboard includes panels for:\n- Query rate and error rate\n- Query latency percentiles (p50, p95, p99)\n- Active connections vs max connections\n- Vector index memory usage\n- Replication lag\n- Database and table sizes\n- Cache hit ratio\n- Checkpoint frequency\n\n## Alerting Rules\n\nCreate Prometheus alerting rules for critical conditions:\n\n```yaml\n# neuraldb-alerts.yml\ngroups:\n - name: neuraldb\n rules:\n\n - alert: NeuralDBConnectionsHigh\n expr: neuraldb_connections_total{state=\"active\"} / neuraldb_connections_max > 0.85\n for: 2m\n labels:\n severity: warning\n annotations:\n summary: \"NeuralDB connections above 85%\"\n description: \"{{ $value | humanizePercentage }} of max connections in use\"\n\n - alert: NeuralDBConnectionsExhausted\n expr: neuraldb_connections_total{state=\"active\"} / neuraldb_connections_max > 0.98\n for: 30s\n labels:\n severity: critical\n annotations:\n summary: \"NeuralDB connections nearly exhausted\"\n\n - alert: NeuralDBHighQueryLatency\n expr: histogram_quantile(0.99, rate(neuraldb_query_duration_seconds_bucket[5m])) > 1.0\n for: 5m\n labels:\n severity: warning\n annotations:\n summary: \"P99 query latency above 1 second\"\n\n - alert: NeuralDBReplicationLagHigh\n expr: neuraldb_replication_lag_seconds > 30\n for: 1m\n labels:\n severity: warning\n annotations:\n summary: \"Replication lag above 30 seconds\"\n\n - alert: NeuralDBDiskSpaceHigh\n expr: (neuraldb_database_size_bytes / disk_total_bytes) > 0.80\n for: 5m\n labels:\n severity: warning\n annotations:\n summary: \"Database storage above 80% capacity\"\n\n - alert: NeuralDBVectorBufferExhausted\n expr: neuraldb_hnsw_index_siz" + }, + { + "file": "pages/ops-scaling.md", + "title": "Scaling", + "section-id": "operations", + "keywords": "scaling, sharding, read replicas, horizontal scaling, capacity planning, performance", + "description": "Scaling NeuralDB horizontally with sharding, read replicas, and capacity planning", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Scaling\n\nNeuralDB is designed to scale horizontally. This page covers adding read replicas for query throughput, sharding for data volume, and capacity planning to avoid resource exhaustion.\n\n## Vertical Scaling (Scale Up)\n\nBefore adding nodes, ensure you have maximised single-node performance:\n\n### Memory\n\nThe biggest lever for NeuralDB performance is memory. Ensure:\n- `vector_buffer` is large enough to hold all active HNSW graphs\n- `shared_buffers` is set to 25% of RAM\n- `work_mem` is appropriate for your query patterns\n\n```sql\n-- Check if vectors are being served from disk (slow) vs memory (fast)\nSELECT index_name, hnsw_graph_size_bytes, hnsw_in_memory\nFROM neuraldb_stat_vector_indexes\nORDER BY hnsw_graph_size_bytes DESC;\n```\n\nIf `hnsw_in_memory = false`, increase `vector_buffer`.\n\n### CPU\n\nVector ANN searches are CPU-bound. Enable parallel query:\n\n```ini\nmax_parallel_workers_per_gather = 8\nmax_parallel_workers = 16\n```\n\n```sql\n-- Allow parallel ANN queries for large tables\nSET max_parallel_workers_per_gather = 8;\nSELECT * FROM large_table ORDER BY embedding <=> $1 LIMIT 10;\n```\n\n### Storage I/O\n\nUse NVMe SSDs with high IOPS. Configure the OS:\n\n```bash\n# Increase read-ahead for sequential I/O\nsudo blockdev --setra 1024 /dev/nvme0n1\n\n# Use deadline/mq-deadline I/O scheduler\necho \"mq-deadline\" | sudo tee /sys/block/nvme0n1/queue/scheduler\n\n# Disable transparent huge pages (reduces latency variability)\necho never | sudo tee /sys/kernel/mm/transparent_hugepage/enabled\n```\n\n## Read Replicas\n\nAdd read replicas to distribute query load.\n\n### Setting Up Read Replicas\n\nFollow the [Replication guide](config-replication.md) to add replicas. Each replica can independently serve `SELECT` queries, including vector similarity searches.\n\n### Client-Side Read Splitting\n\nConfigure your application to route reads to replicas:\n\n**Python:**\n```python\nfrom neuraldb import NeuralDB\n\nprimary = NeuralDB(\"postgresql://neuraldb:pass@primary:5432/mydb\")\nreplica = NeuralDB(\"postgresql://neuraldb:pass@replica:5432/mydb\")\n\ndef search(query_vector):\n # Read goes to replica\n return replica.query(\"SELECT * FROM docs ORDER BY embedding <=> %s LIMIT 10\", [query_vector])\n\ndef insert(content, embedding):\n # Write goes to primary\n return primary.execute(\"INSERT INTO docs (content, embedding) VALUES (%s, %s)\", [content, embedding])\n```\n\n**Connection string with `target_session_attrs`:**\n```\npostgresql://neuraldb:pass@primary:5432,replica:5432/mydb?target_session_attrs=prefer-standby\n```\n\n### Read Replica Scaling Targets\n\n| Replicas | Approximate peak QPS (1536-dim, 10M vectors) |\n|---------|----------------------------------------------|\n| 1 primary | 8,000 |\n| 1 primary + 2 replicas | 24,000 |\n| 1 primary + 4 replicas | 48,000 |\n| 1 primary + 8 replicas | 96,000 |\n\n## Horizontal Sharding\n\nFor datasets exceeding single-node capacity (>50M vectors or >5 TB), shard across multiple primary nodes.\n\n### Shard Configuration\n\n```sql\n-- Create a sharded cluster (requires NeuralDB Cluster Edition)\nSELECT neuraldb_cluster.init_cluster(\n shards => 8,\n replication_factor => 2\n);\n\n-- Create a sharded table\nCREATE TABLE documents (\n id UUID NOT NULL DEFAULT gen_random_uuid(),\n tenant_id UUID NOT NULL,\n content TEXT,\n embedding VECTOR(1536)\n) SHARD BY tenant_id;\n\n-- Each shard holds ~1/8 of the data\n-- All rows with the same tenant_id are colocated on the same shard\n```\n\n### Cross-Shard Queries\n\nCross-shard queries (where the filter doesn't align with the shard key) are automatically parallelised across shards:\n\n```sql\n-- This query executes on all 8 shards in parallel\nSELECT id, content, 1 - (embedding <=> $1) AS similarity\nFROM documents\nORDER BY embedding <=> $1\nLIMIT 10;\n-- Results are merged and re-ranked by the coordinator\n```\n\nPerformance with 8 shards: near-linear scaling. An 8-shard cluster serves ~8\u00d7 the QPS of a single node for cross-shard searches, with ~20% overhead for coordination.\n\n### Shard Rebalancing\n\nWhen adding new shard nodes, rebalance data:\n\n```sql\n-- Rebalance shards (online, non-blocking)\nSELECT neuraldb_cluster.rebalance_shards();\n\n-- Monitor progress\nSELECT * FROM neuraldb_cluster.rebalance_status;\n```\n\n## Capacity Planning\n\n### Storage Capacity\n\nEstimate required storage:\n\n```\nRow data \u2248 avg_row_size_bytes \u00d7 num_rows \u00d7 1.3 (index overhead)\nVector data \u2248 dimensions \u00d7 4 bytes \u00d7 num_vectors\nHNSW graph \u2248 dimensions \u00d7 4 bytes \u00d7 num_vectors \u00d7 1.3\nWAL \u2248 daily_write_volume \u00d7 wal_retention_days\n\nTotal \u2248 row_data + vector_data + HNSW_graph + WAL + 20% buffer\n```\n\nExample: 100M rows, 1536 dimensions, 500 bytes average row size:\n- Row data: 500B \u00d7 100M \u00d7 1.3 \u2248 **65 GB**\n- Vector data: 1536 \u00d7 4B \u00d7 100M \u2248 **614 GB**\n- HNSW graph: 614 GB \u00d7 1.3 \u2248 **800 GB** (must fit in `vector_buffer`)\n- WAL (7 days): 10 GB/day \u00d7 7 = **70 GB**\n- **Total: ~1.6 TB storage, 800 GB RAM for HNSW**\n\n### Connection Capacity\n\n```\nmax_connections = max_app_connections + pgbouncer_pool_size + replication_slots + 3 (superuser)\n```\n\nFor 500 app connecti" + }, + { + "file": "pages/ops-troubleshooting.md", + "title": "Troubleshooting", + "section-id": "operations", + "keywords": "troubleshooting, errors, diagnostics, FAQ, common problems, debug", + "description": "Common NeuralDB errors, diagnostic techniques, and frequently asked questions", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Troubleshooting\n\nThis page covers common NeuralDB errors and how to diagnose and resolve them.\n\n## Connection Issues\n\n### `FATAL: password authentication failed for user \"neuraldb\"`\n\nThe password is incorrect or the user doesn't exist.\n\n```bash\n# Reset the password as the OS postgres user\nsudo -u neuraldb neuraldb-cli\n```\n\n```sql\nALTER USER neuraldb PASSWORD 'new-password';\n```\n\nCheck `pg_hba.conf` \u2014 ensure the correct authentication method is used for the client's IP address.\n\n### `FATAL: no pg_hba.conf entry for host \"x.x.x.x\", user \"neuraldb\"`\n\nThe client's IP is not in `pg_hba.conf`:\n\n```\n# Add to pg_hba.conf\nhost all all x.x.x.x/32 scram-sha-256\n```\n\nReload: `SELECT pg_reload_conf();`\n\n### `could not connect to server: Connection refused`\n\nNeuralDB is not running on the expected host/port:\n\n```bash\n# Check process\nsystemctl status neuraldb\n# or\nps aux | grep neuraldb\n\n# Check listening port\nss -tlnp | grep 5432\n\n# Check logs\njournalctl -u neuraldb -n 50\n```\n\n### `FATAL: remaining connection slots are reserved for non-replication superuser connections`\n\nAll available connections are consumed. Use PgBouncer to pool connections, or increase `max_connections`:\n\n```sql\n-- Check current connections\nSELECT count(*), state, wait_event_type FROM pg_stat_activity GROUP BY state, wait_event_type;\n\n-- Kill idle connections\nSELECT pg_terminate_backend(pid) FROM pg_stat_activity\nWHERE state = 'idle' AND state_change < NOW() - INTERVAL '10 minutes';\n```\n\n## Vector Query Issues\n\n### Slow Vector Searches\n\nIf vector queries are slow, check whether the HNSW index is being used:\n\n```sql\nEXPLAIN (ANALYZE, BUFFERS)\nSELECT id, embedding <=> '[...]' AS dist\nFROM documents\nORDER BY embedding <=> '[...]'\nLIMIT 10;\n```\n\nLook for `Index Scan using documents_embedding_idx` in the plan. If you see `Seq Scan`, the planner may have decided the index is not beneficial.\n\nCommon causes:\n1. **Missing LIMIT clause**: The planner only uses the HNSW index for `ORDER BY ... LIMIT` queries.\n2. **Too few rows**: For small tables, a sequential scan may be faster.\n3. **HNSW graph not in memory**: Check `SELECT * FROM neuraldb_stat_vector_indexes` \u2014 if `hnsw_in_memory = false`, increase `vector_buffer`.\n4. **ef_search too low**: Increase for better recall at the cost of speed.\n\n```sql\n-- Force index use for debugging\nSET enable_seqscan = off;\nEXPLAIN ANALYZE SELECT ... ORDER BY embedding <=> $1 LIMIT 10;\nSET enable_seqscan = on;\n```\n\n### `ERROR: expected 1536 dimensions, not 768`\n\nThe vector you are inserting has a different number of dimensions than the column definition:\n\n```sql\n-- Check column definition\n\\d documents\n-- embedding column shows VECTOR(1536)\n\n-- You are inserting a 768-dimensional vector \u2014 check your embedding model\n```\n\nEnsure your embedding model is consistent. If you need to change models, you must re-embed all existing data.\n\n### Low Recall on ANN Queries\n\nIf approximate queries are not returning expected results:\n\n```sql\n-- Increase ef_search for higher recall\nSET hnsw.ef_search = 200;\nSELECT id, content, 1 - (embedding <=> $1) AS similarity\nFROM documents\nORDER BY embedding <=> $1\nLIMIT 10;\n```\n\nCompare against exact search:\n\n```sql\nSET neuraldb.vector_scan = 'exact';\nSELECT id, content FROM documents ORDER BY embedding <=> $1 LIMIT 10;\n```\n\nIf exact search finds results that approximate search misses, increase `ef_search` or rebuild the index with a higher `m` value.\n\n## Performance Issues\n\n### High Memory Usage\n\n```sql\n-- Check vector index memory consumption\nSELECT index_name, pg_size_pretty(hnsw_graph_size_bytes) AS graph_memory\nFROM neuraldb_stat_vector_indexes\nORDER BY hnsw_graph_size_bytes DESC;\n\n-- Check for shared_buffers usage\nSELECT name, setting, unit FROM pg_settings\nWHERE name IN ('shared_buffers', 'vector_buffer', 'work_mem');\n```\n\n### Disk Space Exhaustion\n\n```sql\n-- Identify large tables and indexes\nSELECT tablename, pg_size_pretty(pg_total_relation_size(tablename::regclass)) AS size\nFROM pg_tables WHERE schemaname = 'public'\nORDER BY pg_total_relation_size(tablename::regclass) DESC\nLIMIT 20;\n\n-- Check WAL accumulation (often caused by idle replication slots)\nSELECT slot_name, active, pg_size_pretty(pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn)) AS lag\nFROM pg_replication_slots\nORDER BY pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn) DESC;\n```\n\nDrop inactive replication slots:\n\n```sql\nSELECT pg_drop_replication_slot('orphaned_slot_name');\n```\n\n### Long-Running Queries\n\n```sql\n-- Find queries running longer than 30 seconds\nSELECT pid, now() - pg_stat_activity.query_start AS duration, query, state\nFROM pg_stat_activity\nWHERE state != 'idle'\n AND (now() - pg_stat_activity.query_start) > INTERVAL '30 seconds'\nORDER BY duration DESC;\n\n-- Terminate a specific query\nSELECT pg_cancel_backend(pid); -- send SIGINT (graceful)\nSELECT pg_terminate_backend(pid); -- send SIGTERM (forceful)\n```\n\n## Replication Issues\n\n### Replication Lag Growing\n\n```sql\n-- Check lag on primary\nSELECT client_addr, state, s" + }, + { + "file": "pages/sdk-go.md", + "title": "Go SDK", + "section-id": "client-sdks", + "keywords": "Go, Golang, SDK, client, connection pool, query builder, pgx", + "description": "The NeuralDB Go SDK \u2014 installation, connection pooling, and vector query builder", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Go SDK\n\nThe NeuralDB Go SDK provides idiomatic Go bindings built on top of `pgx`, the high-performance PostgreSQL driver for Go.\n\n## Installation\n\n```bash\ngo get github.com/neuraldb/neuraldb-go\n```\n\nRequires Go 1.21+.\n\n## Connecting\n\n### Single Connection\n\n```go\npackage main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n\n \"github.com/neuraldb/neuraldb-go\"\n)\n\nfunc main() {\n ctx := context.Background()\n\n client, err := neuraldb.Connect(ctx, \"postgresql://neuraldb:password@localhost:5432/mydb\")\n if err != nil {\n log.Fatal(\"connection failed:\", err)\n }\n defer client.Close(ctx)\n\n var count int64\n err = client.QueryRow(ctx, \"SELECT COUNT(*) FROM documents\").Scan(&count)\n if err != nil {\n log.Fatal(err)\n }\n fmt.Println(\"Documents:\", count)\n}\n```\n\n### Connection Pool (Recommended)\n\n```go\nimport (\n \"github.com/neuraldb/neuraldb-go\"\n \"github.com/jackc/pgx/v5/pgxpool\"\n)\n\nfunc NewPool(ctx context.Context) (*neuraldb.Pool, error) {\n config, err := pgxpool.ParseConfig(os.Getenv(\"NEURALDB_URL\"))\n if err != nil {\n return nil, err\n }\n\n config.MaxConns = 20\n config.MinConns = 5\n config.MaxConnIdleTime = 30 * time.Minute\n config.MaxConnLifetime = time.Hour\n\n return neuraldb.NewPool(ctx, config)\n}\n```\n\n## Working with Vectors\n\n### Defining Vector Types\n\n```go\npackage main\n\nimport \"github.com/neuraldb/neuraldb-go/types\"\n\n// Create a vector from a float32 slice\nv := types.NewVector([]float32{0.023, -0.187, 0.412})\n\n// Create from float64 (auto-converted)\nv2 := types.NewVectorFromFloat64([]float64{0.023, -0.187, 0.412})\n\n// Access the underlying data\nfloats := v.Slice() // []float32\ndims := v.Dims() // int\n```\n\n### Inserting a Document with a Vector\n\n```go\ntype Document struct {\n ID string\n Content string\n Embedding types.Vector\n}\n\nfunc InsertDocument(ctx context.Context, pool *neuraldb.Pool, doc Document) error {\n _, err := pool.Exec(ctx,\n `INSERT INTO documents (id, content, embedding) VALUES ($1, $2, $3)`,\n doc.ID, doc.Content, doc.Embedding,\n )\n return err\n}\n```\n\n### Vector Similarity Search\n\n```go\nfunc SemanticSearch(ctx context.Context, pool *neuraldb.Pool, queryEmbedding []float32, limit int) ([]SearchResult, error) {\n qv := types.NewVector(queryEmbedding)\n\n rows, err := pool.Query(ctx, `\n SELECT id, content, 1 - (embedding <=> $1) AS similarity\n FROM documents\n WHERE embedding IS NOT NULL\n ORDER BY embedding <=> $1\n LIMIT $2\n `, qv, limit)\n if err != nil {\n return nil, fmt.Errorf(\"query failed: %w\", err)\n }\n defer rows.Close()\n\n var results []SearchResult\n for rows.Next() {\n var r SearchResult\n err := rows.Scan(&r.ID, &r.Content, &r.Similarity)\n if err != nil {\n return nil, err\n }\n results = append(results, r)\n }\n\n return results, rows.Err()\n}\n```\n\n## Query Builder\n\nThe SDK includes an optional query builder for type-safe query construction:\n\n```go\nimport \"github.com/neuraldb/neuraldb-go/qb\"\n\n// Build a hybrid query\nquery, args := qb.New().\n Select(\"id\", \"name\", \"price\").\n Expr(\"1 - (embedding <=> $?) AS similarity\", queryVector).\n From(\"products\").\n Where(qb.Eq(\"category\", \"electronics\")).\n Where(qb.GTE(\"price\", 50)).\n Where(qb.LTE(\"price\", 500)).\n Where(qb.GT(\"stock\", 0)).\n OrderByExpr(\"embedding <=> $?\", queryVector).\n Limit(20).\n Build()\n\nrows, err := pool.Query(ctx, query, args...)\n```\n\n## Batch Operations\n\n```go\nimport \"github.com/jackc/pgx/v5\"\n\nfunc BatchInsert(ctx context.Context, pool *neuraldb.Pool, docs []Document) error {\n batch := &pgx.Batch{}\n\n for _, doc := range docs {\n batch.Queue(\n `INSERT INTO documents (content, embedding) VALUES ($1, $2)`,\n doc.Content, doc.Embedding,\n )\n }\n\n results := pool.SendBatch(ctx, batch)\n defer results.Close()\n\n for range docs {\n _, err := results.Exec()\n if err != nil {\n return fmt.Errorf(\"batch insert error: %w\", err)\n }\n }\n\n return results.Close()\n}\n```\n\n## Transactions\n\n```go\nfunc TransactionalInsert(ctx context.Context, pool *neuraldb.Pool, docs []Document) error {\n return pool.BeginTxFunc(ctx, pgx.TxOptions{\n IsoLevel: pgx.ReadCommitted,\n }, func(tx pgx.Tx) error {\n for _, doc := range docs {\n _, err := tx.Exec(ctx,\n `INSERT INTO documents (content, embedding) VALUES ($1, $2)`,\n doc.Content, doc.Embedding,\n )\n if err != nil {\n return err // auto-rolled back by BeginTxFunc\n }\n }\n\n _, err := tx.Exec(ctx,\n `UPDATE stats SET doc_count = doc_count + $1`,\n len(docs),\n )\n return err\n })\n}\n```\n\n## Scanning Results into Structs\n\n```go\nimport \"github.com/jackc/pgx/v5/pgxscan\"\n\ntype Product struct {\n ID string `db:\"id\"`\n Name strin" + }, + { + "file": "pages/sdk-javascript.md", + "title": "JavaScript SDK", + "section-id": "client-sdks", + "keywords": "JavaScript, TypeScript, SDK, Node.js, browser, npm, client", + "description": "The NeuralDB JavaScript/TypeScript SDK for Node.js and browser environments", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# JavaScript SDK\n\nThe NeuralDB JavaScript SDK provides a fully typed client for Node.js and browser environments. It is built on `pg` (node-postgres) for Node.js and a custom HTTP adapter for edge and browser environments.\n\n## Installation\n\n```bash\nnpm install @neuraldb/client\n# or\nyarn add @neuraldb/client\n# or\npnpm add @neuraldb/client\n```\n\n## Basic Setup\n\n### Node.js\n\n```typescript\nimport { NeuralDB } from '@neuraldb/client';\n\nconst client = new NeuralDB({\n connectionString: process.env.NEURALDB_URL!,\n // or individual options:\n host: 'localhost',\n port: 5432,\n user: 'neuraldb',\n password: 'password',\n database: 'mydb',\n ssl: { rejectUnauthorized: true },\n});\n\nawait client.connect();\n```\n\n### Edge / Serverless (HTTP mode)\n\nFor Cloudflare Workers, Vercel Edge, and browser environments, use the HTTP adapter:\n\n```typescript\nimport { NeuralDB, HttpAdapter } from '@neuraldb/client';\n\nconst client = new NeuralDB({\n adapter: new HttpAdapter({\n url: process.env.NEURALDB_HTTP_URL!,\n apiKey: process.env.NEURALDB_API_KEY!,\n }),\n});\n```\n\n### Connection Pool\n\n```typescript\nimport { NeuralDBPool } from '@neuraldb/client';\n\nconst pool = new NeuralDBPool({\n connectionString: process.env.NEURALDB_URL!,\n max: 20,\n idleTimeoutMillis: 30000,\n connectionTimeoutMillis: 5000,\n});\n```\n\n## Executing Queries\n\n```typescript\n// Simple query \u2014 returns QueryResult\nconst result = await client.query('SELECT id, content FROM documents LIMIT 10');\nconst rows = result.rows; // typed as any[]\n\n// Parameterised query\nconst { rows } = await client.query<{ id: string; content: string }>(\n 'SELECT id, content FROM documents WHERE source = $1 LIMIT $2',\n ['web-scraper', 20]\n);\n```\n\n## Vector Operations\n\n### Inserting Vectors\n\n```typescript\nimport { toVector } from '@neuraldb/client';\n\nconst embedding = [0.023, -0.187, 0.412, /* 1536 values */];\n\nawait client.query(\n 'INSERT INTO documents (content, embedding) VALUES ($1, $2)',\n ['My document content', toVector(embedding)]\n);\n```\n\n### Similarity Search\n\n```typescript\nimport OpenAI from 'openai';\nimport { toVector } from '@neuraldb/client';\n\nconst openai = new OpenAI();\n\nasync function semanticSearch(query: string, limit = 10) {\n const embeddingResponse = await openai.embeddings.create({\n model: 'text-embedding-3-small',\n input: query,\n });\n\n const queryVector = embeddingResponse.data[0].embedding;\n\n const { rows } = await client.query<{\n id: string;\n content: string;\n similarity: number;\n }>(\n `SELECT id, content, 1 - (embedding <=> $1) AS similarity\n FROM documents\n WHERE embedding IS NOT NULL\n ORDER BY embedding <=> $1\n LIMIT $2`,\n [toVector(queryVector), limit]\n );\n\n return rows;\n}\n```\n\n### Hybrid Search\n\n```typescript\nasync function hybridSearch(query: string, filters: Record, limit = 10) {\n const queryVector = await generateEmbedding(query);\n\n const conditions: string[] = [];\n const params: unknown[] = [toVector(queryVector)];\n\n Object.entries(filters).forEach(([key, value]) => {\n params.push(value);\n conditions.push(`${key} = $${params.length}`);\n });\n\n const whereClause = conditions.length > 0\n ? 'WHERE ' + conditions.join(' AND ')\n : '';\n\n params.push(limit);\n\n const { rows } = await client.query(\n `SELECT id, content, 1 - (embedding <=> $1) AS similarity\n FROM documents\n ${whereClause}\n ORDER BY embedding <=> $1\n LIMIT $${params.length}`,\n params\n );\n\n return rows;\n}\n```\n\n## High-Level Document API\n\nThe SDK includes a higher-level document management API:\n\n```typescript\nimport { DocumentStore } from '@neuraldb/client';\n\nconst store = new DocumentStore(client, {\n table: 'documents',\n embeddingColumn: 'embedding',\n contentColumn: 'content',\n embeddingModel: {\n provider: 'openai',\n model: 'text-embedding-3-small',\n apiKey: process.env.OPENAI_API_KEY!,\n },\n});\n\n// Add documents (auto-generates embeddings)\nawait store.add([\n { content: 'First document', metadata: { source: 'web' } },\n { content: 'Second document', metadata: { source: 'pdf' } },\n]);\n\n// Search\nconst results = await store.search('query text', {\n limit: 10,\n filter: { source: 'web' },\n minSimilarity: 0.7,\n});\n\n// Delete\nawait store.delete({ filter: { source: 'web' } });\n```\n\n## Transactions\n\n```typescript\nconst pgClient = await pool.connect();\n\ntry {\n await pgClient.query('BEGIN');\n\n await pgClient.query(\n 'INSERT INTO documents (content, embedding) VALUES ($1, $2)',\n ['Content', toVector(embedding)]\n );\n\n await pgClient.query(\n 'UPDATE stats SET doc_count = doc_count + 1 WHERE id = $1',\n [statsId]\n );\n\n await pgClient.query('COMMIT');\n} catch (error) {\n await pgClient.query('ROLLBACK');\n throw error;\n} finally {\n pgClient.release();\n}\n```\n\n## Streaming Results\n\nFor large result sets, stream rows to avoid loading all data into memory:\n\n```typescript\nconst stream = client.queryStream(\n 'SELECT id, content, embedding FROM documents WHERE source = $1',\n ['web-sc" + }, + { + "file": "pages/sdk-python.md", + "title": "Python SDK", + "section-id": "client-sdks", + "keywords": "Python, SDK, client, connection, CRUD, vector operations, psycopg", + "description": "Installing and using the NeuralDB Python SDK \u2014 connection, CRUD, and vector operations", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# Python SDK\n\nThe NeuralDB Python SDK provides a high-level client for Python applications. It is built on top of `psycopg3` (the PostgreSQL adapter) with NeuralDB-specific helpers for vector operations, embedding generation, and batch ingestion.\n\n## Installation\n\n```bash\npip install neuraldb\n# or\npip install neuraldb[asyncio] # includes async support\npip install neuraldb[all] # includes all optional extras\n```\n\n### Requirements\n\n- Python 3.10+\n- libpq (PostgreSQL client library)\n\nOn Ubuntu: `sudo apt install libpq-dev`\nOn macOS: `brew install libpq`\n\n## Connecting\n\n### Synchronous Client\n\n```python\nfrom neuraldb import NeuralDB\n\n# From connection string\nclient = NeuralDB(\"postgresql://neuraldb:password@localhost:5432/mydb\")\n\n# From parameters\nclient = NeuralDB(\n host=\"localhost\",\n port=5432,\n user=\"neuraldb\",\n password=\"password\",\n database=\"mydb\",\n sslmode=\"require\",\n)\n\n# Context manager (auto-closes connection)\nwith NeuralDB(\"postgresql://...\") as client:\n result = client.query(\"SELECT 1\")\n```\n\n### Async Client\n\n```python\nimport asyncio\nfrom neuraldb import AsyncNeuralDB\n\nasync def main():\n async with AsyncNeuralDB(\"postgresql://neuraldb:password@localhost/mydb\") as client:\n result = await client.query(\"SELECT 1\")\n print(result)\n\nasyncio.run(main())\n```\n\n### Connection Pool\n\n```python\nfrom neuraldb import NeuralDBPool\n\npool = NeuralDBPool(\n \"postgresql://neuraldb:password@localhost/mydb\",\n min_size=5,\n max_size=20,\n)\n\nwith pool.acquire() as client:\n result = client.query(\"SELECT COUNT(*) FROM documents\")\n```\n\n## Schema Operations\n\n```python\n# Create a table with a vector column\nclient.execute(\"\"\"\n CREATE TABLE IF NOT EXISTS documents (\n id UUID PRIMARY KEY DEFAULT gen_random_uuid(),\n content TEXT NOT NULL,\n source TEXT,\n embedding VECTOR(1536),\n created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()\n )\n\"\"\")\n\n# Create a vector index\nclient.execute(\"\"\"\n CREATE INDEX IF NOT EXISTS documents_embedding_idx\n ON documents USING hnsw (embedding vector_cosine_ops)\n WITH (m = 16, ef_construction = 64)\n\"\"\")\n```\n\n## CRUD Operations\n\n### Insert\n\n```python\nfrom neuraldb import Vector\n\n# Insert with pre-computed embedding\nclient.execute(\n \"INSERT INTO documents (content, source, embedding) VALUES (%s, %s, %s)\",\n (\"My document content\", \"web-scraper\", Vector([0.023, -0.187, 0.412, ...]))\n)\n\n# Insert many (batched for efficiency)\ndocs = [\n (\"Content A\", \"source-1\", Vector([...])),\n (\"Content B\", \"source-2\", Vector([...])),\n (\"Content C\", \"source-1\", Vector([...])),\n]\nclient.executemany(\n \"INSERT INTO documents (content, source, embedding) VALUES (%s, %s, %s)\",\n docs\n)\n```\n\n### Query\n\n```python\n# Standard query \u2014 returns list of Row objects\nrows = client.query(\"SELECT id, content FROM documents WHERE source = %s\", (\"web-scraper\",))\n\nfor row in rows:\n print(row[\"id\"], row[\"content\"])\n\n# As dicts\nrows = client.query(\n \"SELECT * FROM documents LIMIT 10\",\n row_factory=\"dict\"\n)\n\n# As named tuples\nrows = client.query(\n \"SELECT id, content FROM documents LIMIT 10\",\n row_factory=\"namedtuple\"\n)\n```\n\n### Vector Search\n\n```python\nimport openai\n\n# Generate query embedding\nquery_text = \"high-performance wireless headphones\"\nquery_embedding = openai.embeddings.create(\n model=\"text-embedding-3-small\",\n input=query_text\n).data[0].embedding\n\n# Semantic search\nresults = client.query(\"\"\"\n SELECT id, content, 1 - (embedding <=> %s) AS similarity\n FROM documents\n WHERE embedding IS NOT NULL\n ORDER BY embedding <=> %s\n LIMIT 10\n\"\"\", (Vector(query_embedding), Vector(query_embedding)))\n\nfor row in results:\n print(f\"{row['similarity']:.3f}: {row['content'][:100]}\")\n```\n\n### Using the High-Level Search API\n\n```python\nfrom neuraldb import VectorSearch\n\nsearcher = VectorSearch(client, table=\"documents\", embedding_column=\"embedding\")\n\nresults = searcher.search(\n query_vector=query_embedding,\n limit=10,\n filters={\"source\": \"web-scraper\"},\n metric=\"cosine\",\n)\n```\n\n### Update\n\n```python\nclient.execute(\n \"UPDATE documents SET content = %s, embedding = %s WHERE id = %s\",\n (\"Updated content\", Vector(new_embedding), doc_id)\n)\n```\n\n### Delete\n\n```python\nclient.execute(\"DELETE FROM documents WHERE id = %s\", (doc_id,))\n```\n\n## Transactions\n\n```python\nwith client.transaction():\n client.execute(\"INSERT INTO documents (content, embedding) VALUES (%s, %s)\", (content, Vector(embedding)))\n client.execute(\"UPDATE stats SET count = count + 1\")\n # Auto-commits on exit, rolls back on exception\n```\n\nExplicit control:\n\n```python\nwith client.transaction() as txn:\n try:\n client.execute(\"INSERT ...\")\n client.execute(\"UPDATE ...\")\n txn.commit()\n except Exception:\n txn.rollback()\n raise\n```\n\n## Bulk Ingestion\n\nFor high-throughput ingestion, use the `BulkIngestor`:\n\n```python\nfrom neuraldb import BulkIngestor\n\ningestor = BulkIngestor(\n client,\n " + }, + { + "file": "pages/sdk-rest.md", + "title": "REST API", + "section-id": "client-sdks", + "keywords": "REST API, HTTP, endpoints, authentication, JSON, API", + "description": "NeuralDB REST API reference \u2014 all endpoints, authentication headers, and response formats", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "# REST API\n\nNeuralDB provides an HTTP REST API for environments where a direct database connection is not practical (browser clients, third-party integrations, webhooks). The REST API is a thin HTTP wrapper over NQL.\n\n## Base URL\n\n```\nhttps://your-neuraldb-host:8080/api/v1\n```\n\nFor NeuralDB Cloud:\n\n```\nhttps://[cluster-id].cloud.neuraldb.io/api/v1\n```\n\n## Authentication\n\nAll REST API requests require an API key in the `Authorization` header:\n\n```\nAuthorization: Bearer ndb_live_your_api_key_here\n```\n\nOr as a query parameter (less secure, avoid in production):\n\n```\n?api_key=ndb_live_your_api_key_here\n```\n\nCreate API keys in the NeuralDB CLI or the Cloud dashboard:\n\n```bash\nneuraldb-cli -c \"SELECT neuraldb_apikeys.create_key(label => 'my-app', role => 'app_readwrite')\"\n```\n\n## Query Endpoint\n\nExecute any NQL query:\n\n### `POST /query`\n\n```http\nPOST /api/v1/query\nContent-Type: application/json\nAuthorization: Bearer ndb_live_...\n\n{\n \"query\": \"SELECT id, content, 1 - (embedding <=> $1) AS similarity FROM documents ORDER BY embedding <=> $1 LIMIT 5\",\n \"params\": [[0.023, -0.187, 0.412]],\n \"database\": \"mydb\"\n}\n```\n\n**Response:**\n\n```json\n{\n \"rows\": [\n { \"id\": \"uuid-1\", \"content\": \"First document\", \"similarity\": 0.923 },\n { \"id\": \"uuid-2\", \"content\": \"Second document\", \"similarity\": 0.891 }\n ],\n \"rowCount\": 2,\n \"fields\": [\n { \"name\": \"id\", \"dataTypeID\": 2950 },\n { \"name\": \"content\", \"dataTypeID\": 25 },\n { \"name\": \"similarity\", \"dataTypeID\": 701 }\n ],\n \"executionTimeMs\": 3.2\n}\n```\n\n### Query Parameters\n\n| Parameter | Type | Required | Description |\n|-----------|------|----------|-------------|\n| `query` | string | Yes | NQL query string |\n| `params` | array | No | Query parameters (replaces $1, $2, ...) |\n| `database` | string | No | Target database (default: `neuraldb`) |\n| `timeout_ms` | integer | No | Query timeout in milliseconds |\n| `explain` | boolean | No | Return query execution plan |\n\n## Document Endpoints\n\nHigher-level CRUD endpoints for document management:\n\n### `POST /collections/:collection/documents`\n\nInsert documents (auto-generates embeddings if configured):\n\n```http\nPOST /api/v1/collections/my_docs/documents\nContent-Type: application/json\nAuthorization: Bearer ndb_live_...\n\n{\n \"documents\": [\n {\n \"content\": \"NeuralDB is an AI-native database\",\n \"metadata\": { \"source\": \"blog\", \"category\": \"technology\" }\n }\n ],\n \"embedding_model\": \"openai/text-embedding-3-small\"\n}\n```\n\n**Response:**\n\n```json\n{\n \"inserted\": 1,\n \"ids\": [\"550e8400-e29b-41d4-a716-446655440000\"]\n}\n```\n\n### `POST /collections/:collection/search`\n\nVector similarity search:\n\n```http\nPOST /api/v1/collections/my_docs/search\nContent-Type: application/json\nAuthorization: Bearer ndb_live_...\n\n{\n \"query\": \"AI-native database for semantic search\",\n \"limit\": 10,\n \"min_similarity\": 0.7,\n \"filters\": {\n \"category\": \"technology\"\n },\n \"embedding_model\": \"openai/text-embedding-3-small\",\n \"include_metadata\": true\n}\n```\n\n**Response:**\n\n```json\n{\n \"results\": [\n {\n \"id\": \"550e8400-e29b-41d4-a716-446655440000\",\n \"content\": \"NeuralDB is an AI-native database\",\n \"similarity\": 0.956,\n \"metadata\": { \"source\": \"blog\", \"category\": \"technology\" }\n }\n ],\n \"query_embedding_model\": \"openai/text-embedding-3-small\",\n \"latency_ms\": 2.8\n}\n```\n\n### `GET /collections/:collection/documents/:id`\n\nRetrieve a document by ID:\n\n```http\nGET /api/v1/collections/my_docs/documents/550e8400-e29b-41d4-a716-446655440000\nAuthorization: Bearer ndb_live_...\n```\n\n### `PUT /collections/:collection/documents/:id`\n\nUpdate a document:\n\n```http\nPUT /api/v1/collections/my_docs/documents/550e8400-...\nContent-Type: application/json\nAuthorization: Bearer ndb_live_...\n\n{\n \"content\": \"Updated content\",\n \"metadata\": { \"source\": \"blog\", \"updated\": true },\n \"regenerate_embedding\": true\n}\n```\n\n### `DELETE /collections/:collection/documents/:id`\n\nDelete a document:\n\n```http\nDELETE /api/v1/collections/my_docs/documents/550e8400-...\nAuthorization: Bearer ndb_live_...\n```\n\n## Batch Operations\n\n### `POST /collections/:collection/documents/batch`\n\nInsert up to 1,000 documents in a single request:\n\n```http\nPOST /api/v1/collections/my_docs/documents/batch\nContent-Type: application/json\n\n{\n \"documents\": [\n { \"content\": \"Document 1\", \"metadata\": {} },\n { \"content\": \"Document 2\", \"metadata\": {} }\n ]\n}\n```\n\n## Error Responses\n\nAll errors return a consistent JSON structure:\n\n```json\n{\n \"error\": {\n \"code\": \"QUERY_ERROR\",\n \"message\": \"relation \\\"nonexistent\\\" does not exist\",\n \"detail\": \"ERROR: relation \\\"nonexistent\\\" does not exist at character 15\",\n \"status\": 400\n }\n}\n```\n\n### Error Codes\n\n| HTTP Status | Error Code | Description |\n|-------------|-----------|-------------|\n| 400 | `QUERY_ERROR` | Invalid NQL query |\n| 400 | `VALIDATION_ERROR` | Request body validation failed |\n| 401 | `UNAUTHORIZED` | Missing or invalid API key |\n| 403 | `FORBIDDEN` | Insufficient role permissions |\n| 404 | `NOT_FOUND` | Document o" + } +] \ No newline at end of file diff --git a/sample-sites/neuraldb-docs/theme.yml b/sample-sites/neuraldb-docs/theme.yml new file mode 100644 index 0000000..d6fead7 --- /dev/null +++ b/sample-sites/neuraldb-docs/theme.yml @@ -0,0 +1,66 @@ +# mdcms theme โ€” default +# Edit colours, fonts, and layout here. See docs for full reference. + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# Colours +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +light: + accent: "#2563EB" + background: "#FFFFFF" + nav-background: "#F8FAFC" + text: "#1E293B" + text-muted: "#64748B" + +dark: + accent: "#60A5FA" + background: "#0F172A" + nav-background: "#1E293B" + text: "#F1F5F9" + text-muted: "#94A3B8" + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# Semantic colours +# Used by callout tags (info, warning, success, error). +# Choose values that work on both light and dark backgrounds. +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +colours-semantic: + info: "#2563EB" + warning: "#D97706" + success: "#16A34A" + error: "#DC2626" + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# Callout defaults +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +callouts: + info: + icon: info + primary-colour: "#2563EB" + background-colour: "#2563EB" + warning: + icon: warning + primary-colour: "#D97706" + background-colour: "#D97706" + success: + icon: success + primary-colour: "#16A34A" + background-colour: "#16A34A" + error: + icon: error + primary-colour: "#DC2626" + background-colour: "#DC2626" + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# Typography +# Format: "provider:Font Name:weight" (provider: bunny | google) +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +font-body: "bunny:Noto Sans:400" +font-heading: "bunny:Noto Sans:700" +font-size: 1.0 # unitless multiplier (1.0 = 16px base) +line-height: 1.7 # unitless multiplier + +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# Layout +# โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +main-width: 80em +nav-width: 20em diff --git a/sample-sites/techpulse/assets/images/ai-coding.jpg b/sample-sites/techpulse/assets/images/ai-coding.jpg new file mode 100644 index 0000000..6cabf11 Binary files /dev/null and b/sample-sites/techpulse/assets/images/ai-coding.jpg differ diff --git a/sample-sites/techpulse/assets/images/hero.jpg b/sample-sites/techpulse/assets/images/hero.jpg new file mode 100644 index 0000000..246902f Binary files /dev/null and b/sample-sites/techpulse/assets/images/hero.jpg differ diff --git a/sample-sites/techpulse/assets/images/open-source.jpg b/sample-sites/techpulse/assets/images/open-source.jpg new file mode 100644 index 0000000..b82f0be Binary files /dev/null and b/sample-sites/techpulse/assets/images/open-source.jpg differ diff --git a/sample-sites/techpulse/config.yml b/sample-sites/techpulse/config.yml new file mode 100644 index 0000000..9adf220 --- /dev/null +++ b/sample-sites/techpulse/config.yml @@ -0,0 +1,6 @@ +# mdcms v0.3 | DO NOT REMOVE THIS COMMENT +sitename: TechPulse +sitedescription: Independent technology news and analysis +navigation: topbar +search: true +footer: "ยฉ 2026 TechPulse Media. All rights reserved." diff --git a/sample-sites/techpulse/index.html b/sample-sites/techpulse/index.html new file mode 100644 index 0000000..1466873 --- /dev/null +++ b/sample-sites/techpulse/index.html @@ -0,0 +1,2784 @@ + + + + + + + +MD-CMS + + + + + + + + + + + + + + +
+ + + + + + diff --git a/sample-sites/techpulse/nav.yml b/sample-sites/techpulse/nav.yml new file mode 100644 index 0000000..0529bfa --- /dev/null +++ b/sample-sites/techpulse/nav.yml @@ -0,0 +1,43 @@ +# nav.yml โ€” generated by mdcms.py +sections: + - code: site + defaultname: Site + sort: 100 + pagesvisibility: visible + +pages: + - file: pages/home.md + title: Home + section-id: site + sort: 100 + variants: [en] + titles: + en: Home + - file: pages/about.md + title: About TechPulse + section-id: site + sort: 110 + variants: [en] + titles: + en: About TechPulse + - file: pages/newsletter.md + title: Newsletter + section-id: site + sort: 120 + variants: [en] + titles: + en: Newsletter + - file: pages/topics.md + title: Topics + section-id: site + sort: 130 + variants: [en] + titles: + en: Topics + - file: pages/archive.md + title: Archive + section-id: site + sort: 140 + variants: [en] + titles: + en: Archive diff --git a/sample-sites/techpulse/pages/about.md b/sample-sites/techpulse/pages/about.md new file mode 100644 index 0000000..102aeab --- /dev/null +++ b/sample-sites/techpulse/pages/about.md @@ -0,0 +1,69 @@ +--- +title: About TechPulse +sort: 110 +section-id: site +keywords: about, team, editorial policy, mission, advertising +description: About TechPulse โ€” our mission, editorial team, and advertising policy +language: en +--- + +# About TechPulse + +TechPulse was founded in 2021 with a specific frustration: technology journalism had largely split into two camps. On one side, outlet after outlet that published whatever the PR teams of major companies sent them, dressed up with a few quotes and published as fast as possible to win search clicks. On the other, a small number of paywalled publications charging enterprise prices and writing primarily for investors and executives. + +Neither camp was doing right by the people who actually build software for a living โ€” the engineers, architects, open-source maintainers, and technical leads who need accurate, technically literate reporting to make good decisions. That's the gap TechPulse was built to fill. + +## Our Mission + +We exist to provide independent, technically grounded reporting on the technology industry. That means: + +- **We do our own research.** Survey data attributed to TechPulse was collected by us. When we cite a study, we've read it, not just its press release. +- **We talk to practitioners.** Our primary sources are engineers, not PR contacts. We regularly talk to CTOs, open-source maintainers, and developers at companies shipping at scale. +- **We distinguish opinion from reporting.** Analysis pieces are clearly labelled. When we don't know something, we say so. +- **We correct our errors.** Mistakes happen. When they do, we correct them transparently and don't quietly edit posts without acknowledgement. + +## The Team + +### Maya Osei โ€” Editor-in-Chief + +Maya joined TechPulse as founding editor after seven years covering the technology industry, including five years as a senior technology reporter at a national newspaper. Her background is in economics, which shapes her particular interest in the business models and incentive structures underlying technical decisions. At TechPulse she oversees all editorial, writes the weekly analysis column, and leads the annual developer survey. She's based in London. + +Maya's work has a particular focus on funding trends, the economics of developer platforms, and the gap between what gets funded and what gets built. Her 2024 series on AI startup revenue โ€” separating genuine ARR from creative accounting โ€” was cited widely and sparked a useful industry conversation about what "revenue" actually means at the pre-product stage. + +### Raj Patel โ€” AI and Developer Tools Correspondent + +Raj came to journalism after six years as a software engineer, including time at a mid-size SaaS company and a stint at a developer tools startup. That background is his greatest asset: he can read a benchmark, understands the difference between microbenchmarks and production performance, and has the context to spot when a company's claims don't hold up technically. + +At TechPulse, Raj covers AI and machine learning with a focus on practical deployment โ€” not capability demos, but what actually works when you ship it into production. He also covers developer tools broadly: runtimes, languages, databases, and the infrastructure layer. His analysis of AI coding assistants and their real-world productivity impact in 2024 became one of TechPulse's most-read pieces of that year. + +Raj is based in Amsterdam and holds a BSc in Computer Science from the University of Manchester. + +### Clara Winthorpe โ€” Open Source and Infrastructure Editor + +Clara has covered open source for eight years, initially for a specialist publication before joining TechPulse at launch. She sits on the programme committee of a major open-source conference and is a regular speaker on open-source sustainability โ€” the question of who funds the software the world runs on, and what happens when that funding dries up. + +Her 2024 deep-dive on open-source sustainability following the xz backdoor incident remains essential reading for anyone thinking seriously about software supply chain risk. She also covers infrastructure: Kubernetes, cloud platforms, WebAssembly, and the ongoing evolution of how companies run software. + +Clara is based in Berlin. + +## Editorial Independence + +TechPulse is funded by reader subscriptions and a small number of clearly disclosed sponsor arrangements. We do not accept money from companies in exchange for coverage. We do not accept press trips, gifts, or other in-kind value from companies we cover. If a company sponsors a newsletter edition, that arrangement is disclosed in the newsletter and does not influence editorial decisions. + +## Advertising Policy + +We accept a limited number of sponsorships per year from companies whose products are relevant to our audience. Our policies: + +1. **No sponsored articles.** Sponsorships are clearly delineated sections of newsletter editions, not articles. +2. **No coverage guarantees.** A sponsorship does not entitle a company to a review, a mention, or any editorial attention. +3. **No conflicts of interest.** Editors who own equity in or have personal relationships with a company disclose this and do not write about that company without another editor's oversight. +4. **Full disclosure.** Any financial relationship between TechPulse and a company mentioned in our coverage is disclosed at the time of publication. + +If you have questions about our advertising policy, or if you'd like to discuss a sponsorship, contact us at advertising@techpulse.example. + +## Contact + +- **Editorial:** editor@techpulse.example +- **Tips and sources:** tips@techpulse.example (Signal available on request) +- **Advertising:** advertising@techpulse.example +- **Corrections:** corrections@techpulse.example diff --git a/sample-sites/techpulse/pages/archive.md b/sample-sites/techpulse/pages/archive.md new file mode 100644 index 0000000..c3d7a5c --- /dev/null +++ b/sample-sites/techpulse/pages/archive.md @@ -0,0 +1,20 @@ +--- +title: Archive +sort: 140 +section-id: site +keywords: archive, all articles, technology news archive +description: The complete TechPulse article archive โ€” all posts in reverse chronological order +language: en +--- + +# Article Archive + +The complete TechPulse archive, updated with every new publication. Use the search function to find articles on a specific topic, or browse by scrolling below. Our coverage spans AI and machine learning, open source, developer tools, startups and funding, cloud infrastructure, and security. + +Articles are listed in reverse chronological order. Our most recent coverage appears first. + +```mdcms +posts-datetime-reversechronological +limit: 10 +paginate: yes +``` diff --git a/sample-sites/techpulse/pages/home.md b/sample-sites/techpulse/pages/home.md new file mode 100644 index 0000000..c7aecd5 --- /dev/null +++ b/sample-sites/techpulse/pages/home.md @@ -0,0 +1,48 @@ +--- +title: Home +sort: 100 +section-id: site +keywords: technology news, developer tools, AI, open source, startups +description: TechPulse โ€” independent technology news and analysis for developers and tech professionals +language: en +--- + +![TechPulse โ€” Independent Technology News](assets/images/hero.jpg) + +# Welcome to TechPulse + +TechPulse is an independent technology publication built for developers and tech-adjacent professionals who want more than press releases. We cover the stories that matter: the real impact of new tools on the people who use them, the economics behind open-source sustainability, the gap between venture capital hype and shipping product. + +Founded in 2021, we operate without advertising from the companies we cover. Our revenue comes from readers โ€” through subscriptions and a small number of transparent sponsorships from companies whose products we genuinely respect. That independence matters. When we say an AI coding assistant improves productivity by 40%, we've done the research. When we say a funding round deserves skepticism, we've talked to the people actually building in that space. + +## What We Cover + +We focus on five areas where signal-to-noise is lowest: + +**Artificial Intelligence and Machine Learning** โ€” not the breathless announcements, but the actual capability changes, the deployment realities, and the business models that are working versus those still searching for product-market fit. + +**Open Source** โ€” the infrastructure that powers the modern internet, the people maintaining it for free, and the ongoing question of who bears the cost. We've been covering open-source sustainability before it became fashionable. + +**Developer Tools** โ€” languages, runtimes, frameworks, cloud platforms. We run benchmarks, talk to engineers at companies shipping at scale, and try to cut through the hype cycles. + +**Startups and Funding** โ€” who's building what, where the money is going, and what the data says about which bets are paying off. We're skeptical of unicorn valuations and interested in profitability. + +**Security** โ€” supply chain vulnerabilities, compliance realities, and the ongoing struggle to ship secure software in a world moving faster than its defences. + +## Our Team + +Maya Osei leads TechPulse as editor-in-chief, bringing ten years of technology journalism experience. Raj Patel covers AI and developer tools. Clara Winthorpe specialises in open source and infrastructure. Together they publish several pieces per week, with longer investigations appearing monthly. + +## Latest Articles + +```mdcms +posts-datetime-reversechronological +limit: 10 +paginate: yes +``` + +## Why Subscribe? + +The internet is drowning in technology content. Most of it is written by people who have never shipped production code, never managed an engineering team, or have a financial interest in the outcome they're describing. We aim to be different โ€” technically literate, financially independent, and honest about uncertainty. + +[Subscribe to the TechPulse newsletter](pages/newsletter.md) to get our weekly digest. Or dive straight into our [topic guides](pages/topics.md) to find coverage in the areas you care about most. diff --git a/sample-sites/techpulse/pages/newsletter.md b/sample-sites/techpulse/pages/newsletter.md new file mode 100644 index 0000000..a6f98c6 --- /dev/null +++ b/sample-sites/techpulse/pages/newsletter.md @@ -0,0 +1,72 @@ +--- +title: Newsletter +sort: 120 +section-id: site +keywords: newsletter, subscribe, weekly digest, email +description: Subscribe to the TechPulse weekly newsletter โ€” technology news and analysis in your inbox +language: en +--- + +# The TechPulse Newsletter + +Every Saturday morning, the TechPulse newsletter lands in subscriber inboxes with a carefully curated week in technology. No clickbait. No press release reprints. Just the stories that mattered, explained with context. + +## What You Get + +**The Weekly Digest** is our flagship newsletter. Each edition includes: + +- **Three featured articles** โ€” our most significant pieces of the week, with brief editorial notes on why they matter +- **Five links worth reading** โ€” the best technology reporting from across the web, with our take on each +- **The data point of the week** โ€” one number from a study, survey, or dataset that deserves attention, with full context +- **Reader question** โ€” each week we pick one reader question and answer it properly, in 200-400 words +- **What we're watching** โ€” a short note on stories we're tracking that haven't fully developed yet + +**The Monthly Deep-Dive** is a longer-form newsletter for subscribers who want to go deeper. Past editions have included: + +- A 3,000-word analysis of the AI coding assistant market and what the productivity data actually shows +- A detailed breakdown of cloud provider pricing changes and their real impact on startup burn rates +- A reader survey on Kubernetes adoption patterns across company sizes and the surprising findings + +## How to Subscribe + +Enter your email address below and choose your subscription tier. The basic newsletter is free. Full access โ€” including the monthly deep-dive and our complete article archive โ€” is available to paying subscribers. + +**Free tier:** Weekly digest delivered every Saturday morning. + +**Subscriber tier (ยฃ6/month or ยฃ55/year):** Everything in the free tier plus the monthly deep-dive, full archive access, and the ability to submit reader questions for consideration. + +*To subscribe, visit techpulse.example/subscribe โ€” email subscriptions are managed via our newsletter platform.* + +## Past Newsletters + +Here are some recent editions to give you a sense of what to expect: + +**May 10, 2026 โ€” The Agentic Turn** +This week we looked at the gap between "AI agents" as described in funding announcements and AI agents as they actually function in enterprise deployments. Spoiler: the gap is large, and the interesting failures are instructive. + +**April 26, 2026 โ€” The Platform Wars, Round Three** +GitHub vs. GitLab vs. everything else. With our developer survey data showing consolidation accelerating, we asked whether the microtools era is genuinely ending and what that means for startups in the space. + +**April 12, 2026 โ€” Open Source After the Funding Boom** +A look at what happened to the open-source projects that received significant corporate investment in 2021-2023 and whether that investment translated into sustainable maintenance models. + +**March 29, 2026 โ€” The Real Cost of Technical Debt** +We surveyed 400 engineers about their estimates of technical debt's impact on their teams. The numbers are worse than most CTOs publicly admit. + +**March 15, 2026 โ€” Benchmarks Are Broken (And Everyone Knows It)** +A deep look at how AI and database benchmarks are constructed, who funds them, and why the numbers in press releases rarely survive contact with production workloads. + +## Reader Feedback + +*"The TechPulse newsletter is the only tech email I actually read on the day it arrives. The signal-to-noise ratio is exceptional."* +โ€” Engineering Manager, Series B startup + +*"Raj's coverage of AI tools is the most technically honest I've found. He actually understands what he's writing about."* +โ€” Senior Software Engineer, fintech + +*"The monthly deep-dives are worth the subscription price alone. The AI coding assistant analysis saved our team from a decision we would have regretted."* +โ€” CTO, 40-person software company + +## Unsubscribing + +You can unsubscribe at any time. Every newsletter edition includes a one-click unsubscribe link at the bottom. We don't make you confirm twice or explain yourself. diff --git a/sample-sites/techpulse/pages/topics.md b/sample-sites/techpulse/pages/topics.md new file mode 100644 index 0000000..cc256c0 --- /dev/null +++ b/sample-sites/techpulse/pages/topics.md @@ -0,0 +1,68 @@ +--- +title: Topics +sort: 130 +section-id: site +keywords: AI, open source, developer tools, startups, cloud, security, hardware, technology coverage +description: An overview of TechPulse's coverage areas โ€” AI/ML, open source, developer tools, startups, cloud, security, and hardware +language: en +--- + +# Topics We Cover + +TechPulse maintains sustained coverage across seven core areas of technology. We don't chase every story โ€” we focus on the beats where we have the expertise and the source networks to report with genuine depth. Here is what you can expect from each area. + +## Artificial Intelligence and Machine Learning + +AI is the defining technology story of the 2020s, and it is also the most heavily hyped. Our AI coverage, led primarily by Raj Patel, tries to separate what is real from what is marketing. We cover capability advances and their genuine significance, but we focus more heavily on deployment realities: what actually works when organisations ship AI into production, what fails, what the failure modes look like, and what the business models look like for companies building in this space. + +We cover large language models, inference infrastructure, AI coding tools, AI agents and automation, and the policy and regulatory environment around AI development. We are particularly interested in the gap between benchmark performance and real-world performance โ€” a gap that turns out to be large and underreported. + +**Recent coverage:** Chain-of-thought models and their enterprise adoption patterns, open source LLM benchmarks, AI agents in production. + +## Open Source + +Open source software is the infrastructure of the modern internet, and yet the economics of how it gets maintained remain deeply broken. Clara Winthorpe leads our open source coverage with a depth of community knowledge that comes from years of genuine participation โ€” she is not an observer, she is a contributor. + +We cover major open source projects, governance fights, sustainability funding, the business models of open-source companies, and the security landscape. The xz backdoor incident crystallised questions about supply chain security that we had been covering for years. We will keep covering them. + +**Recent coverage:** The 2025 Kubernetes fatigue survey, Deno v3 Node.js compatibility, software supply chain security progress report. + +## Developer Tools + +Languages, runtimes, frameworks, editors, databases, CI/CD pipelines, build systems. The developer tooling landscape moves fast and has enormous amounts of money flowing through it, which produces a lot of noise. We try to cut through that noise by running our own tests, talking to engineers at companies shipping real software, and applying a scepticism that is hard-earned. + +We are particularly interested in the gap between what gets VC attention and what developers actually use. The tools that win in the long run are not always the ones with the best marketing, and we try to track both the hype cycle and the actual adoption curve. + +**Recent coverage:** SQLite's rise in edge computing, WebAssembly components and the component model, Deno v3 benchmarks. + +## Startups and Funding + +The business of technology shapes which ideas get built. We cover venture capital funding, startup formation, acquisitions, and the IPO market with a critical eye โ€” we are interested in what the numbers actually say, not what founders and investors want them to say. + +Maya Osei leads this coverage, bringing an economics background that shapes how we think about unit economics, runway, revenue quality, and the incentive structures that drive decisions at funded companies. We are deeply sceptical of the framing that surrounds most funding announcements and try to give readers what they need to assess the claims. + +**Recent coverage:** AI startup funding vs. revenue in H1 2024, tech funding in 2025, developer platform consolidation. + +## Cloud and Infrastructure + +The cloud computing landscape has matured into something complex and expensive. AWS, GCP, and Azure dominate, but a growing ecosystem of platform alternatives is competing for developer mindshare. We cover this landscape from a practitioner perspective โ€” what does it actually cost to run things, what are the operational tradeoffs, and which platforms are delivering genuine value versus vendor lock-in dressed up as convenience. + +We also cover the emerging edge computing infrastructure, serverless architectures, and the ongoing tension between managed services and self-hosted alternatives. + +**Recent coverage:** Platform engineering vs. DevOps, Kubernetes fatigue and what teams are doing instead. + +## Security + +Software security is a beat that rewards sustained attention. We cover vulnerability disclosures, supply chain security, security tooling, enterprise security posture, and the policy environment. We try to report on security in a way that is useful to practitioners โ€” explaining the technical details accurately, giving appropriate severity context, and avoiding both panic and dismissiveness. + +We have a particular focus on software supply chain security, which has moved from a niche concern to a central issue in the wake of incidents like SolarWinds, Log4Shell, and xz. + +**Recent coverage:** Software supply chain security progress report, SBOM adoption, sigstore and related tooling. + +## Hardware + +We maintain a modest but sustained hardware coverage programme, focused on the chips and systems that matter to software developers: server processors, AI accelerators, and the evolving relationship between hardware capabilities and the software written to exploit them. We do not cover consumer electronics. + +--- + +*Browse all coverage using the [Archive](pages/archive.md) or [search](/) for a specific topic.* diff --git a/sample-sites/techpulse/posts/2024-03-12-llm-coding-assistants.md b/sample-sites/techpulse/posts/2024-03-12-llm-coding-assistants.md new file mode 100644 index 0000000..7071a16 --- /dev/null +++ b/sample-sites/techpulse/posts/2024-03-12-llm-coding-assistants.md @@ -0,0 +1,67 @@ +--- +title: "The Real Impact of AI Coding Assistants on Developer Productivity" +created: 2024-03-12 09:00 +author: Raj Patel +keywords: AI coding assistants, GitHub Copilot, developer productivity, code quality, security vulnerabilities +description: A study of 500 developers reveals a 40% productivity gain from AI coding tools โ€” but the picture is more complicated than that number suggests. +--- + +![AI Coding Tools in Practice](assets/images/ai-coding.jpg) + +The claim had been circulating for months before anyone tested it rigorously: AI coding assistants make developers significantly more productive. GitHub cited a 55% productivity increase in one controlled study. Other vendors published numbers ranging from 30% to 70%. The figures were eye-catching enough that engineering managers had started asking their teams to adopt tools they barely understood. + +We wanted to know what the numbers looked like in practice, with real codebases and real deadlines. Over three months, TechPulse conducted a study with 500 developers across 40 companies โ€” from eight-person startups to engineering organisations with several thousand employees. The headline number is real: developers using AI coding assistants completed assigned tasks approximately 40% faster than developers working without them. But the story behind that number is considerably more complicated. + +## What the 40% Number Actually Measures + +The productivity gain is real, but it is narrow. The tasks where AI assistants shine are tasks that involve writing code that follows patterns the model has seen many times: implementing a standard CRUD endpoint, writing unit tests for a function, converting data between formats, generating boilerplate for a new module. These are real tasks that occupy real developer time, and getting through them faster is genuinely valuable. + +The 40% improvement collapses significantly on tasks that require architectural reasoning, debugging complex interactions, or working with novel or unusual codebases. Several engineering leads we interviewed noted that junior developers using AI assistance were completing simple tasks quickly but struggling more with integration and debugging โ€” skills that develop partly through the friction of writing code by hand. + +"The tool makes the easy stuff faster," one senior engineer at a fintech company told us. "The hard stuff is still hard. Sometimes it's harder, because the AI has generated three hundred lines of plausible-looking code that has subtle bugs in it, and now I have to find them." + +## Code Quality Concerns + +Every organisation in our study that had been using AI coding tools for more than six months reported concerns about code quality. The pattern was consistent: AI-generated code tends to pass automated tests (partly because AI tools are good at writing tests to match the code they just wrote), but it tends to have more subtle architectural issues, more duplication, and higher cyclomatic complexity than code written by experienced developers from scratch. + +We reviewed 3,000 pull requests across six companies that had adopted AI coding tools, comparing them against a baseline period before adoption. Code review times increased by 23% on average, and the fraction of pull requests that required significant rework before merging increased from 18% to 29%. Engineering managers who had expected AI tools to reduce code review burden found the opposite. + +One particularly striking finding: AI tools generated code that cited non-existent library functions in approximately 4% of completions โ€” a phenomenon the AI community calls "hallucination" but that engineers working with production code describe less charitably. In most cases this was caught during compilation or testing, but not always. + +## Security Vulnerabilities in AI-Generated Code + +The security picture is the most concerning finding in our study. A research team at Stanford published a paper in 2023 showing that developers using GitHub Copilot were more likely to introduce security vulnerabilities than developers without assistance. Our study found similar patterns. + +Working with a security consultancy, we reviewed AI-generated code across fifteen repositories and identified security issues at a rate roughly 1.8x higher than the baseline codebases. The most common issues were SQL injection vulnerabilities, insecure random number generation, improper input validation, and hardcoded credentials โ€” all classic beginner-level security errors that experienced developers have learned to avoid. + +The problem is not simply that AI generates insecure code. It is that AI generates insecure code that looks plausible and confident, which is harder to catch than obviously amateurish code. Several CTOs we interviewed noted that they had tightened security review requirements after adopting AI coding tools, which partially offset the productivity gains. + +"We're faster at getting to review, but review itself is more expensive," one CTO said. "The net is positive, but not as positive as the raw productivity numbers suggest." + +## Developer Dependency and Skill Development + +Four of the five most senior developers we interviewed โ€” people with 15 or more years of experience โ€” expressed concern about what AI coding tools are doing to skill development at the junior level. The concern is not luddite: all of them use the tools themselves. The concern is structural. + +Learning to code involves making mistakes, finding bugs, building mental models through failure. AI tools smooth over that friction. Junior developers who use AI assistants heavily may be writing more code per hour than their predecessors, but they may also be building shallower mental models of how that code actually works. Several engineering leads reported that junior developers who had learned primarily through AI-assisted coding had difficulty debugging issues that the AI could not solve โ€” which is to say, the hard cases. + +"They can write a React component, but they don't really know what's happening inside it," one engineering manager told us. "Five years ago, a junior developer who couldn't explain what they'd written would be a red flag. Now it's become normal, and I'm not sure that's good." + +## Practical Recommendations + +Our recommendation for engineering organisations is neither uncritical adoption nor reflexive rejection. AI coding assistants are real productivity tools with real limitations. Used thoughtfully, they save time on repetitive tasks and reduce the cognitive cost of context-switching. Used carelessly, they create security debt and development practices that don't scale. + +Specific recommendations based on our study: + +**Do not remove mandatory code review.** The temptation to treat AI-generated code as more reliable than it is will cost you in the medium term. + +**Invest in security review tooling.** Static analysis and SAST tools should be running on all AI-generated code, and security training should be updated to cover AI-specific vulnerability patterns. + +**Think carefully about junior developer onboarding.** The productivity gains are lowest for junior developers, and the skill development concerns are highest. Consider structured periods of work without AI assistance. + +**Track quality metrics, not just velocity.** If your only measurement is pull request throughput, you will optimise for throughput. Track defect rates, review times, and rework rates as well. + +The 40% productivity gain is real. So are the costs. Engineering organisations that acknowledge both are better positioned to capture the benefits while managing the risks. + +--- + +*Study methodology: 500 developers across 40 companies, surveyed December 2023 through February 2024. Task completion data collected via controlled task assignments; code quality data collected via pull request analysis with developer consent. Full methodology available on request.* diff --git a/sample-sites/techpulse/posts/2024-05-20-open-source-sustainability.md b/sample-sites/techpulse/posts/2024-05-20-open-source-sustainability.md new file mode 100644 index 0000000..6ada037 --- /dev/null +++ b/sample-sites/techpulse/posts/2024-05-20-open-source-sustainability.md @@ -0,0 +1,61 @@ +--- +title: "Open Source Sustainability Crisis: Who Pays for the Infrastructure?" +created: 2024-05-20 14:00 +author: Clara Winthorpe +keywords: open source, sustainability, xz backdoor, OpenSSF, Sovereign Tech Fund, funding +description: The xz backdoor incident exposed what many already knew โ€” the open source infrastructure powering global commerce is maintained by a handful of burned-out volunteers. Who should pay for it? +--- + +![Open Source Infrastructure](assets/images/open-source.jpg) + +In late March 2024, a lone security researcher named Andres Freund noticed something odd while investigating slow SSH logins on his Debian machine. After several hours of careful investigation, he discovered that a utility called xz โ€” a compression library used by nearly every Linux distribution on the planet โ€” had been deliberately backdoored by a person who had spent nearly two years systematically building trust in the project. + +The attacker, who used the alias "Jia Tan," had contributed carefully to the project, built relationships with the exhausted maintainer, gradually taken on more responsibility, and ultimately introduced malicious code in a new release. Had Freund not been unusually attentive, the backdoor would have shipped in the next Debian stable release, potentially giving the attacker root access to millions of systems. + +The incident was a near-miss, and near-misses have a way of clarifying structural problems. The xz backdoor was not primarily a story about one clever attacker. It was a story about a maintainer who was exhausted, burned out, and clearly being manipulated by someone who had identified the soft spot in the infrastructure of global computing. It was a story about a critical piece of software being maintained by one person, effectively alone, for years. + +## The Scale of the Problem + +The problem is not new, but the xz incident gave it a face. The Log4Shell vulnerability in 2021 was another crystallising moment โ€” a critical flaw in a library maintained by a handful of volunteers and used by an enormous fraction of enterprise Java applications. The maintainers were not paid by the companies whose software depended on their work. They were volunteers. + +A 2022 census by the Harvard Institute for Quantitative Social Science and the Linux Foundation found that a significant proportion of the most-depended-upon open source packages were maintained by one or two people. The most popular packages on npm and PyPI were maintained by individuals who, in many cases, had day jobs that had nothing to do with open source. + +The economic pattern is easy to understand and hard to solve. Open source is a public good โ€” software that, once created, can be used by anyone without reducing its availability to others. Public goods are chronically underprovided by markets, because the value they generate is not captured by the people who provide them. Companies that build products on top of open source software capture enormous value while contributing very little back to the infrastructure that makes it possible. + +This is not a moral judgment about those companies. The incentive structure simply does not reward contribution. If you are a startup trying to survive, spending engineering time on upstream contributions is expensive and the benefit is diffuse and long-term. You take the open source library, use it, and move on. + +## What Is Being Done + +Several organisations have taken serious steps to address the problem, though none of them at the scale the problem requires. + +**The Open Source Security Foundation (OpenSSF)** was established in 2020 under the Linux Foundation umbrella with a mission to improve the security of the open source supply chain. After Log4Shell, it received a significant injection of funding from major technology companies โ€” $150 million pledged at a White House summit in 2022. The OpenSSF has funded important work including security reviews of critical packages, developer training, and tooling for software supply chain security. Critics argue it remains under-resourced and too focused on tooling and standards rather than directly funding maintainers. + +**The Sovereign Tech Fund**, established by the German Federal Government, takes a different approach: it directly funds maintenance work on specific open source projects with demonstrated public-interest importance. The funding is structured as contracts, which means maintainers are paid for the work they do. The approach is less scalable than an industry-wide levy but more direct in its impact. + +**GitHub Sponsors and Open Collective** provide mechanisms for individuals and organisations to fund open source maintainers directly. These platforms have enabled some maintainers to earn meaningful income from their work, but the amounts are rarely sufficient to make open source maintenance a full-time job for the people maintaining the most critical infrastructure. + +**Corporate open source programmes** at companies like Google, Microsoft, and Red Hat fund significant open source development, but primarily on projects that serve their own strategic interests. The correlation between corporate open source investment and public infrastructure importance is imperfect. + +## Three Models for a Solution + +The open source sustainability problem has generated considerable debate about structural solutions. Three models receive the most serious attention. + +**The infrastructure levy model** proposes requiring companies above a certain revenue threshold that derive benefit from open source software to contribute a percentage of their revenue โ€” or their open source benefit โ€” to a pooled fund. The pooled fund would then distribute money to projects based on dependency data and criticality scores. The model is attractive in its comprehensiveness and its alignment with the public-goods economics of open source. The challenge is implementation: who decides which projects are critical, who administers the fund, and how do you compel contribution internationally? + +**The procurement mandate model** proposes requiring government and critical infrastructure organisations to demonstrate that the open source software in their supply chains is adequately funded and maintained โ€” similar to how procurement rules already require vendors to demonstrate security practices. This creates a demand-side pressure on companies using open source software in government contracts. The weakness is scope: government procurement represents only a fraction of open source usage. + +**The foundation consolidation model** argues that rather than trying to fund individual maintainers, the solution is to consolidate important open source projects under well-resourced foundations that have sustainable funding models. The Apache Software Foundation and the Linux Foundation represent versions of this model. Critics argue that not all valuable open source projects can or should become foundation projects, and that foundation governance introduces its own bureaucracy and risk. + +## What the xz Incident Actually Tells Us + +The xz backdoor incident tells us something important that the sustainability discussion often misses: the risk is not just that unmaintained projects become insecure through neglect. The risk is that burned-out maintainers are actively targeted by sophisticated actors who understand that exhaustion and isolation make people vulnerable to manipulation. + +The person who attacked xz did not exploit a code vulnerability. They exploited a social vulnerability โ€” a maintainer who was clearly struggling, who had been expressing burnout in public for months, and who was susceptible to the apparent helpfulness of a patient, skilled contributor. The attack required patience, social engineering, and a long-term strategy. It was a state-level or near-state-level operation targeting the weakest link in critical software infrastructure. + +No amount of tooling addresses that threat directly. Only sustainable, funded maintenance โ€” maintainers who have colleagues, who are not working alone under financial pressure, who have the time and support to be discerning about contributors โ€” reduces that risk meaningfully. + +The xz incident was a near-miss. The next one may not be. + +--- + +*Clara Winthorpe covers open source and infrastructure at TechPulse. She contributed to documentation for two of the affected packages.* diff --git a/sample-sites/techpulse/posts/2024-07-08-rust-linux-kernel.md b/sample-sites/techpulse/posts/2024-07-08-rust-linux-kernel.md new file mode 100644 index 0000000..0246bb6 --- /dev/null +++ b/sample-sites/techpulse/posts/2024-07-08-rust-linux-kernel.md @@ -0,0 +1,57 @@ +--- +title: "Rust in the Linux Kernel: One Year Later" +created: 2024-07-08 10:30 +author: Clara Winthorpe +keywords: Rust, Linux kernel, kernel drivers, systems programming, memory safety, Linus Torvalds +description: One year after the first Rust code landed in the Linux kernel, we assess what has merged, how developers have received it, and what the safety improvements look like in practice. +--- + +When Linus Torvalds merged the initial Rust infrastructure into Linux 6.1 in December 2022, it marked the first time in the kernel's history that a second programming language had been accepted as a peer to C. The decision was not without controversy โ€” some longtime kernel developers questioned the choice, the timeline, and the claimed benefits. Now, roughly eighteen months later, there is enough real-world experience to make a meaningful assessment. + +## What Has Actually Merged + +The scope of Rust in the kernel as of mid-2024 is narrower than some coverage has suggested, but it is growing. The initial merge provided the infrastructure: the Rust toolchain integration, the core abstractions over kernel primitives, and the `rust/` directory in the kernel source tree. Actual Rust drivers and subsystems have followed more gradually. + +The most significant Rust code in the mainline kernel at the time of writing is in the device driver space. The Nova GPU driver โ€” an open-source, Rust-based driver for NVIDIA hardware โ€” was merged in 6.9 after an extensive review period. Several other drivers have merged or are in active review, primarily in the filesystem and networking subsystems where memory safety is most critical. + +Miguel Ojeda, who has led the Rust-for-Linux initiative, provided us with current numbers: as of Linux 6.9, there are approximately 31,000 lines of Rust code in the kernel tree, compared to roughly 27 million lines of C. The Rust code represents about 0.1% of the total, but the trajectory is consistently upward. The rate of Rust submissions has increased with each kernel release cycle. + +## Developer Reception Has Warmed, Slowly + +Early reactions from kernel developers ranged from enthusiastic to hostile. The hostility has softened, though not universally. The most prominent dissenter, Linus's statement that he considered Rust "a nice toy language" has evolved significantly over time โ€” he has become a more cautious but genuine supporter, provided the Rust code meets the same bar as the C code. + +"I don't care what language you write in," Torvalds told a kernel developer conference audience in 2023. "I care that the code is correct, maintainable, and doesn't break anything. Rust can be all of those things. It can also be none of those things. That's a property of the code, not the language." + +Several kernel maintainers we interviewed noted that the quality of Rust submissions has been high. "The people doing Rust kernel work are motivated and careful," said one subsystem maintainer who asked not to be named. "The code I've reviewed has been well-thought-out. My concern is the long-term: who maintains this in five years when the novelty wears off and you need someone to do boring debugging at 2am?" + +The concern about maintainer depth is legitimate. The pool of developers who can write kernel C is large; the pool who can write kernel Rust is currently much smaller. This limits the reviewers available for Rust patches and creates bus-factor risk in subsystems that go Rust-first. + +## Measuring Safety Improvements + +The central claim for Rust in the kernel is safety: Rust's ownership and borrowing system prevents entire classes of memory safety bugs โ€” use-after-free, double-free, data races โ€” that have historically been responsible for a large fraction of kernel security vulnerabilities. + +Measuring this benefit is difficult, because the Rust code is new and has not yet accumulated the years of production exposure needed to generate statistical safety data. What the data does show is that approximately 65-70% of security vulnerabilities in the Linux kernel over the past decade have been memory-safety bugs โ€” buffer overflows, use-after-free errors, and similar issues that the Rust type system prevents at compile time. + +In theory, subsystems written in Rust should not produce this class of vulnerability. In practice, there are caveats. Rust kernel code must frequently use `unsafe` blocks to interact with the C kernel primitives that the Rust code is wrapping. The safety properties hold within the Rust code, but the boundary between Rust and C requires careful handling. Early reviews of kernel Rust code identified several instances where `unsafe` blocks were more expansive than necessary, providing less isolation benefit than the code appeared to offer. + +The longer-term benefit will accrue as more critical code is written in Rust from scratch โ€” code that can maintain Rust's safety invariants more completely. This is a decade-long project, not a near-term transformation. + +## The C Developers' Perspective + +We spoke at length with several kernel developers who remain skeptical, not of Rust in principle, but of the pace and scope of adoption. The concern is not primarily technical; it is ecosystem. + +"The kernel has thirty years of tooling, documentation, and institutional knowledge built around C," one experienced kernel developer told us. "A new contributor who wants to understand a C subsystem can find tutorials, can read the same documentation generations of contributors have read, can use the same debugging tools. For Rust kernel work, they're much more on their own." + +There is also a practical concern about Rust's evolution. The Rust language and compiler change more rapidly than C, and the kernel currently requires a minimum Rust version that is pinned for each kernel release. Managing the Rust toolchain requirement across the long support periods that enterprise Linux distributions depend on is not a solved problem. + +## What Comes Next + +The near-term roadmap for Rust in the kernel is focused on two areas. First, expanding the abstractions library โ€” the safe Rust wrappers over kernel C primitives โ€” to cover more of the kernel API surface. Currently, writing Rust drivers for certain subsystems requires writing `unsafe` C-interface code that should eventually be handled by the abstractions layer. Second, the Nova driver and a small number of other ambitious Rust kernel projects will serve as test cases for whether Rust is viable for large, complex kernel subsystems or only for simpler, self-contained drivers. + +The longer-term trajectory depends on whether the generation of developers who grew up with Rust enters kernel development in significant numbers. If they do, Rust's share of the kernel will grow organically. If kernel development remains primarily the province of experienced C programmers, Rust will remain a promising experiment in a niche. + +Given the sustained trajectory of the past year and a half, the former looks more likely than the latter. + +--- + +*Clara Winthorpe covers open source and infrastructure. She has been following Rust-for-Linux since the project's early days.* diff --git a/sample-sites/techpulse/posts/2024-08-15-startup-ai-funding.md b/sample-sites/techpulse/posts/2024-08-15-startup-ai-funding.md new file mode 100644 index 0000000..0d764cd --- /dev/null +++ b/sample-sites/techpulse/posts/2024-08-15-startup-ai-funding.md @@ -0,0 +1,60 @@ +--- +title: "AI Startup Funding Hits $47B in H1 2024 โ€” But Where's the Revenue?" +created: 2024-08-15 11:00 +author: Maya Osei +keywords: AI funding, startup investment, venture capital, AI revenue, AI startups 2024 +description: AI startups raised $47 billion in the first half of 2024. A detailed look at which categories received it, which companies are generating real revenue, and which are burning cash. +--- + +Forty-seven billion dollars. That is how much venture capital flowed into AI startups in the first half of 2024, according to data compiled by TechPulse from CB Insights, PitchBook, and Crunchbase. To put the number in context: it exceeds the total venture capital raised by the entire US startup ecosystem in H1 2019. In five years, AI has eaten the investment market. + +The question that VCs are asking each other quietly but rarely in public is whether the investment is producing commensurate revenue. Our analysis suggests that the answer is: in some categories, yes; in most, not yet; and in a meaningful subset, probably never. + +## Where the Money Went + +The $47 billion was not distributed evenly. Breaking it down by category: + +**Foundation model companies** received approximately $14.2 billion, dominated by the major rounds at OpenAI ($6.6B at a $157B valuation), Anthropic ($2.75B from Google and Amazon in the period), and xAI ($6B). These are the companies building the large language models that underpin the AI application layer. Their revenue is real and growing: OpenAI's annualised revenue run rate was reported at around $3.4 billion, and Anthropic was reportedly approaching $1 billion ARR. At the valuations being assigned, those are still extraordinary multiples โ€” OpenAI's $157B valuation is roughly 46x its reported ARR. But the revenue is there. + +**AI infrastructure** โ€” chips, cloud AI services, MLOps platforms, inference infrastructure โ€” received approximately $9.3 billion. NVIDIA's extraordinary performance (market cap exceeding $2 trillion for part of the period) is not a startup story, but the infrastructure ecosystem around it is growing fast. The companies in this category are largely generating real revenue, because enterprise demand for AI compute is genuine and growing. + +**Vertical AI applications** โ€” AI applied to specific domains like legal (Harvey, Ironclad), healthcare (Hippocratic, Abridge), finance, HR, and similar โ€” received approximately $12.1 billion. This is the most heterogeneous category. Some vertical AI companies are generating meaningful revenue with strong retention. A significant proportion are still in proof-of-concept territory with enterprise customers and have weak ARR metrics dressed up with LOI pipelines and inflated TCV figures. + +**AI developer tools** โ€” code assistants, AI-enhanced IDEs, automated testing, and similar โ€” received approximately $7.4 billion. GitHub Copilot's success has created a rush of competitors, and while the category is real, saturation risk is high. The companies most likely to survive in the long run are either best-of-breed (which requires genuine technical differentiation) or embedded deeply enough in enterprise workflows to have real switching costs. + +**AI agents and automation** received approximately $4 billion, mostly in smaller rounds. This is the category with the widest gap between funding narrative and actual product. The vision โ€” autonomous AI agents that handle complex multi-step tasks โ€” is compelling and may be realised in some form over the coming years. Current products are not there yet. Several well-funded companies in this space are reporting primarily "pilot" customers rather than paying ARR. + +## The Revenue Quality Problem + +The most revealing conversations we had for this piece were with LP-facing analysts at major venture firms โ€” the people who have to actually account for portfolio performance to their investors. Off the record, the picture they described is concerning. + +"We have companies reporting 'ARR' that is really annualised MRR from customers who are on 90-day pilots," one told us. "We have companies reporting revenue that includes non-recurring professional services. We have companies where the top three customers represent 70% of ARR and all three are still in their free trial periods. The definition inflation is extraordinary." + +Enterprise AI adoption is real but slow. The pattern we see repeatedly in our reporting is: initial excitement, a pilot programme, promising early results in narrow use cases, and then a long pause while the enterprise works out whether and how to integrate the tool into actual workflows. This process takes longer than the AI hype cycle suggests. Companies with six-month-old products are being valued on the assumption that the adoption curve will compress dramatically; the evidence from the enterprise software market suggests it will not. + +Investors who understand the enterprise software market well are applying steeper haircuts to AI ARR than the headline valuations suggest. Several VCs told us they are internally modelling AI companies at 60-70% of reported ARR for purposes of portfolio assessment. + +## Which Categories Are Most Exposed + +The categories most exposed to a correction are those where the competition is intense, the technology is not genuinely differentiated, and the customers are not yet committed: + +AI writing tools, AI image generation consumer apps, and general-purpose AI productivity tools face the worst dynamics. The foundation models are commoditising rapidly, which squeezes the margin on applications that are thin wrappers over them. Customer retention data for AI writing tools, in particular, is poor โ€” users sign up, use them enthusiastically for a month, and churn at high rates. + +Agents and automation face a different problem: the technology is not yet reliable enough for the use cases being pitched. The gap between "impressive demo" and "production-reliable at scale" in agentic AI is substantial, and enterprise customers who have been burned by AI pilots that worked in controlled settings but failed in production are becoming more sceptical. + +## What the Data Suggests About the Next 18 Months + +Our assessment: the AI investment cycle has several more quarters of momentum, but the rationalization is coming. The companies that will survive the correction are those with: + +- Real, auditable ARR from enterprise customers who have moved past pilot stages +- Genuine technical differentiation that is not easily replicated by a new model release +- Unit economics that work without assuming continued capital infusion +- Customer retention rates that suggest genuine product-market fit + +By those criteria, the foundation model companies, the infrastructure layer, and a subset of vertical AI companies are on solid ground. A significant portion of the application layer is not. + +The $47 billion will produce lasting value. Not all of it, and not for all the investors who deployed it. + +--- + +*Revenue data based on reported figures and analyst estimates. Funding data compiled from CB Insights, PitchBook, and Crunchbase. All figures are approximations; private company financials are not publicly audited.* diff --git a/sample-sites/techpulse/posts/2024-10-03-sqlite-everywhere.md b/sample-sites/techpulse/posts/2024-10-03-sqlite-everywhere.md new file mode 100644 index 0000000..6881db9 --- /dev/null +++ b/sample-sites/techpulse/posts/2024-10-03-sqlite-everywhere.md @@ -0,0 +1,69 @@ +--- +title: "The SQLite Revolution: How a 25-Year-Old Database Took Over the Cloud" +created: 2024-10-03 09:00 +author: Raj Patel +keywords: SQLite, Cloudflare D1, Turso, libSQL, edge computing, databases, cloud +description: SQLite was designed for embedded systems. Somehow it has become the database of choice for the edge computing era. Here is why, and what its limitations mean for the future. +--- + +SQLite was created by D. Richard Hipp in 2000 for use on guided missile destroyers, where the alternative โ€” a server-based database โ€” was impractical on a ship. It is a library, not a server: the database is a single file, the query engine lives in your process, and there is no network connection to manage. For 24 years, SQLite was the database in your phone, your browser, and your laptop โ€” the invisible infrastructure of the device world. + +Then the cloud industry discovered it. + +In the past two years, SQLite has become the unexpected centrepiece of a new wave of database services, edge computing platforms, and developer tools. Cloudflare D1 runs SQLite at the edge. Turso has built a distributed SQLite service with a fork of the engine. The authors of libSQL, a fork of SQLite that supports extensions and replication, have raised significant venture funding. Bun, the JavaScript runtime, uses SQLite as its built-in database. The pattern is everywhere: developers and platforms are choosing SQLite for its simplicity, reliability, and the extraordinary density of its feature-to-complexity ratio. + +## Why Edge Computing Loves SQLite + +The reason SQLite works so well for edge computing comes down to three properties: it is an in-process library, it has no network overhead, and it produces a single portable file. + +Edge computing workloads run in small, short-lived execution environments โ€” Cloudflare Workers, Deno Deploy, Fastly Compute, and similar platforms that spin up code close to users to reduce latency. These environments have a fundamental problem with traditional databases: you cannot maintain a persistent connection to a remote database if your process might be running in Johannesburg one moment and Sรฃo Paulo the next. Connection pooling becomes complex, latency becomes unpredictable, and the network hop adds overhead that defeats the purpose of edge execution. + +SQLite sidesteps this problem by eliminating the network entirely. The database lives in the same process as the code, queries run at memory speed, and the whole thing can be replicated to multiple edge locations as a file. This is the insight that Cloudflare's D1 team had: if you can replicate a SQLite file efficiently to hundreds of edge locations, you get a globally distributed database with very low read latency and the simplicity of a single-file store. + +## The Cloudflare D1 Architecture + +Cloudflare announced D1 โ€” its edge database service built on SQLite โ€” in 2022, initially in beta. By 2024 it had moved to general availability and was handling a significant fraction of Workers deployments that needed persistence. + +The architectural approach is worth understanding. Each D1 database is a SQLite file. Writes are processed by a primary SQLite instance running in a datacenter. Reads can be served from read replicas โ€” copies of the SQLite file โ€” that are distributed close to users at Cloudflare's edge locations. Replication uses a write-ahead log that is forwarded from the primary to replicas. + +The tradeoffs are real but manageable for many use cases. Read-after-write consistency is eventually consistent: if you write data and immediately read it from a different edge location, you might get stale data. The replication lag is typically measured in milliseconds for nearby regions and hundreds of milliseconds for distant ones. For most web application workloads โ€” displaying content, user profiles, product catalogues โ€” this is acceptable. For workloads that require strong consistency (financial transactions, inventory management, anything where reading stale data causes real problems), it is not. + +## Turso and the libSQL Fork + +Turso occupies an interesting position in the SQLite ecosystem. They have built a distributed SQLite service, but they have also funded the development of libSQL โ€” an open-source fork of SQLite that adds features the original SQLite project has declined to include: replication support, server mode, and extension APIs. + +The SQLite project, under Hipp's stewardship, has been notably conservative about expanding the library's scope. The original design philosophy โ€” a simple, reliable, self-contained library โ€” has been maintained with unusual discipline. Hipp's view is that SQLite should do one thing extremely well and not try to become something it was not designed to be. + +libSQL takes the opposite view: take SQLite's quality and battle-tested storage engine and add the capabilities needed for modern cloud deployments. The extension APIs in libSQL are particularly interesting โ€” they allow embedding vector search, full-text search, and other capabilities as loadable extensions rather than baked-in features. + +Turso raised $32 million in a Series A in 2024, which gives some indication of investor confidence in the approach. Whether libSQL becomes a sustainable open-source project with genuine community support, or whether it remains primarily a Turso-maintained fork, remains to be seen. The tension between the commercial interests of a funded startup and the maintenance of a genuinely open project is a familiar one. + +## Comparison with PlanetScale + +PlanetScale, which built a developer-friendly MySQL-as-a-service with branching and schema migration features, was the darling of the previous wave of developer database tools. It announced a pricing change in 2024 that eliminated its free tier and caused significant developer backlash, followed by a reversal. The episode illustrated both the strength of developer affection for the product and the difficulty of finding sustainable business models in developer infrastructure. + +SQLite-based services have a structural advantage in this comparison: the cost basis for running SQLite at the edge is lower than running a full MySQL-compatible distributed database. This gives SQLite platforms more flexibility in pricing, which is a meaningful competitive advantage in a market where developer adoption depends heavily on having a workable free tier. + +The more fundamental question is whether the use cases overlap. PlanetScale (and MySQL generally) is better suited to applications with complex relational schemas, heavy write workloads, and strong consistency requirements. SQLite-based services are better suited to read-heavy workloads, simpler schemas, and applications where the edge latency benefit justifies the eventual consistency tradeoff. + +## The Limitations Nobody Talks About Enough + +SQLite's concurrency model is its most significant limitation for cloud workloads. SQLite uses file-level locking: only one write can happen at a time to a given database file. For applications with high write concurrency โ€” many users simultaneously writing to the same database โ€” this is a real constraint that no amount of edge distribution fully addresses. + +The services built on SQLite have various mitigations: Cloudflare D1 routes all writes to a primary, Turso does similar things. But the fundamental limitation of the storage engine is that it was not designed for high-concurrency writes, and it shows. Applications that need to handle thousands of concurrent writes per second to a single logical dataset will exhaust SQLite's capabilities. + +For a large class of web applications, this is not a binding constraint. Most web applications are read-heavy, and the read:write ratio for typical content-serving, e-commerce, and user-profile workloads is often 10:1 or higher. In those cases, SQLite's concurrency model is not the bottleneck. + +For the workloads where it is a constraint, the answer is not SQLite-at-the-edge but a distributed, strongly consistent database โ€” which will always come with latency and operational complexity tradeoffs that SQLite does not. + +## The Surprising Thing About SQLite + +The most striking thing about the SQLite story is that it is about engineering quality compounding over time. Hipp has been working on SQLite for 24 years. The project has a test suite with more than 92 million test cases. Every line of code in SQLite is tested; the test code is significantly larger than the library code. This level of investment in correctness is unusual in any software project and exceptional in an open-source library. + +The result is a piece of software that developers trust completely โ€” and that trust, it turns out, is an asset with enormous value in an industry that produces new databases at a rate that makes it impossible to develop equivalent trust in any of them quickly. + +SQLite is winning not because it was designed for cloud computing but because it was designed with extraordinary care, and extraordinary care compounds. + +--- + +*Raj Patel is TechPulse's AI and developer tools correspondent. He has been following the SQLite ecosystem since 2021.* diff --git a/sample-sites/techpulse/posts/2024-11-18-wasm-components.md b/sample-sites/techpulse/posts/2024-11-18-wasm-components.md new file mode 100644 index 0000000..8562909 --- /dev/null +++ b/sample-sites/techpulse/posts/2024-11-18-wasm-components.md @@ -0,0 +1,69 @@ +--- +title: "WebAssembly Components: The Runtime-Agnostic Future of Software" +created: 2024-11-18 13:00 +author: Raj Patel +keywords: WebAssembly, WASM, WASI, component model, ByteCode Alliance, containers, runtime +description: WASI 0.2 and the WebAssembly component model represent a genuinely new approach to software packaging. Here is what it means and who is actually using it. +--- + +In January 2024, the WebAssembly System Interface working group released WASI 0.2 โ€” the second major version of the interface standard that allows WebAssembly programs to interact with the operating system. The release was accompanied by the component model, a specification for how WebAssembly modules can be packaged, composed, and distributed in a way that is independent of the runtime, the language, and the operating system. + +The claims made for WebAssembly components are ambitious: code written in Rust, Go, Python, or any language that compiles to WebAssembly should be composable, portable, and secure, regardless of where it runs. The component model is the mechanism that makes that composability possible in a principled way. + +After a year of working with the specification and talking to teams using it in production, TechPulse can report that the reality is more promising than the hype, more mature than skeptics expected, and still some distance from ubiquity. + +## What the Component Model Actually Solves + +To understand why the component model matters, it helps to understand what it is solving. Traditional software distribution has a language problem: code written in Rust cannot call code written in Python without a foreign function interface (FFI) that is language-specific, fragile, and usually requires careful attention to memory management at the boundary. + +The component model defines a standard interface definition language (WIT โ€” WebAssembly Interface Types) and a standard way of encoding values at the boundary between components. A Rust component and a Python component that both implement the same WIT interface can be composed together by a runtime, with the runtime handling type conversion and memory isolation at the boundary. + +This is similar to what container-based microservices achieve, but at a finer granularity and with much lower overhead. A WebAssembly component is not a full container with its own OS layer; it is a module with a typed interface. Starting time is measured in microseconds, not hundreds of milliseconds. Memory overhead is kilobytes, not megabytes. + +## WASI 0.2: What Changed + +The jump from WASI 0.1 (Preview 1) to WASI 0.2 was significant. WASI 0.1 provided basic POSIX-like capabilities: files, environment variables, clocks, and random numbers. It was enough to run many programs but notably lacked networking โ€” a significant limitation for server-side software. + +WASI 0.2 adds a networking API (wasi-sockets) and, more importantly, the component model plumbing that makes composition possible. Components in WASI 0.2 can import and export typed interfaces, can be composed at link time, and can run in any compliant runtime regardless of where they were compiled. + +The ByteCode Alliance โ€” the industry consortium that coordinates WASI and the component model specification, with membership including Fastly, Microsoft, Google, Intel, and others โ€” shipped supporting tooling in 2024 that made the specification practically usable. The Wasmtime runtime reached production readiness for WASI 0.2 workloads. The `cargo component` toolchain for Rust-based component development became stable. + +## Real-World Adoption: Who Is Using It + +Adoption of WebAssembly components in production is real but concentrated. + +**Fastly** is the most advanced production user. Their Compute product runs customer code as WebAssembly modules at the edge, and they have been pushing the component model as the basis for Compute's plugin ecosystem. Fastly engineers have contributed significantly to the specification and tooling. + +**Fermyon Technologies** built their Spin framework โ€” a developer-friendly tool for building serverless WebAssembly applications โ€” around the component model. Spin has a growing user base and is one of the clearest demonstrations of what component-based development looks like in practice. Fermyon has also been shipping Fermyon Cloud, a managed hosting service for Spin applications. + +**Microsoft** has incorporated WebAssembly components in several Azure services, most notably in their edge networking infrastructure. The details are less public, but multiple Microsoft engineers are active contributors to the component model specification. + +**Shopify** has been evaluating WebAssembly as the execution substrate for their storefront extension system, which needs to run untrusted third-party code in a sandboxed environment. The security properties of WebAssembly โ€” memory isolation, no ambient authority, fine-grained capability grants โ€” make it attractive for this use case. + +## Comparison with Containers + +The inevitable comparison is with Docker containers, which solved the portability and packaging problem for a previous era. Containers won because they gave developers a standard unit of packaging that was runtime-agnostic, reproducible, and composable through orchestration layers. + +WebAssembly components are not a container replacement in the general case. They are a different tool for a different class of problems. Containers provide full OS-level isolation with their own filesystem, network stack, and process tree. WebAssembly components provide CPU-level isolation within a shared process. Containers are right for full applications with complex dependencies. Components are right for plugins, functions, and composable modules where startup overhead, memory cost, and security sandboxing requirements favour a lighter model. + +The most interesting near-term application of WebAssembly components is probably in the extension and plugin ecosystem: giving application developers a safe, performant way to allow untrusted code to run inside their applications without compromising the host. This is the use case that Shopify, Fastly, and a number of other production adopters are building around. + +## What Still Needs Work + +The component model ecosystem has meaningful gaps that will take time to close. + +Language support is uneven. Rust and C/C++ have mature toolchains for producing WebAssembly components. Go's support has improved but still has limitations. Python and JavaScript tooling is functional but produces larger binary sizes due to the need to bundle runtime interpreters. The ecosystem is moving toward "guest toolchains" for major languages, but the work is not complete. + +Debugging and observability tooling lags. Debugging a WebAssembly component through a standard debugger requires additional DWARF extension support that is not uniformly available. Distributed tracing across component boundaries is not standardised. These are solvable problems but they have not been solved yet. + +The component model also has a learning curve that is real and non-trivial. WIT interface design is a new skill; understanding capability-based security for WebAssembly requires new mental models; and the toolchain surface area is larger than it appears. Teams adopting WebAssembly components in 2024 are still early adopters who should expect rough edges. + +## The Longer View + +WebAssembly components represent a genuine attempt to solve a problem โ€” secure, language-agnostic composition of software modules โ€” that no previous technology has fully solved. Containers solved deployment packaging. Function-as-a-service platforms solved some of the same problems but with high startup overhead and coarser granularity. The component model is attempting something more fundamental: a universal substrate for computation that is language-independent, runtime-independent, and platform-independent. + +Whether that ambition is achievable depends on adoption dynamics that are not yet determined. The specification is solid. The tooling is reaching production quality. The early adopters are building real systems. Whether it achieves the penetration needed to become a default rather than a speciality depends on the next two to three years. + +--- + +*This article draws on conversations with engineers at Fastly, Fermyon, and several ByteCode Alliance member companies, as well as the author's own work with Spin and Wasmtime.* diff --git a/sample-sites/techpulse/posts/2024-12-05-developer-survey-2024.md b/sample-sites/techpulse/posts/2024-12-05-developer-survey-2024.md new file mode 100644 index 0000000..bb11f47 --- /dev/null +++ b/sample-sites/techpulse/posts/2024-12-05-developer-survey-2024.md @@ -0,0 +1,99 @@ +--- +title: "TechPulse Developer Survey 2024: 3,000 Respondents, Key Findings" +created: 2024-12-05 10:00 +author: Maya Osei +keywords: developer survey 2024, programming languages, AI tools, remote work, salary, developer burnout +description: Results from our annual survey of 3,000 developers โ€” language popularity, AI adoption, salary data, remote work trends, and burnout rates. +--- + +Every year since 2022, TechPulse has run an independent developer survey. This year 3,047 developers completed our questionnaire โ€” the largest sample we have collected. Respondents came from 61 countries, with the largest groups from the United States (34%), United Kingdom (11%), Germany (8%), India (7%), and Canada (6%). + +The survey ran for three weeks in October and November 2024. What follows are our key findings. + +## Language Popularity and Usage + +Python has extended its lead as the most widely-used language in our survey, with 67% of respondents reporting that they use Python for some professional work. The growth is driven primarily by AI/ML work: Python's dominance in the machine learning ecosystem has pulled a generation of developers who might otherwise have stuck to Java or JavaScript into regular Python usage. + +**Regularly used languages (respondents could select multiple):** +- Python: 67% +- JavaScript/TypeScript: 64% +- Rust: 19% (up from 13% in 2023) +- Go: 31% +- Java: 38% +- C#: 27% +- C/C++: 22% +- Kotlin: 14% +- Swift: 9% +- Ruby: 8% + +TypeScript's adoption has now surpassed plain JavaScript for new projects among respondents with more than five years of experience. Eighty-one percent of JavaScript developers in our survey are using TypeScript for at least some projects, up from 71% in 2023. + +Rust's continued growth is striking. It has moved from a curious experiment to a serious production language in the span of four years. The communities driving adoption are systems programming (kernel, embedded, network infrastructure), web assembly, and increasingly, backend web services where its memory safety and performance characteristics are valued. + +## AI Tool Adoption + +This is the finding that dominates the conversation this year. Seventy-three percent of respondents are using some form of AI coding assistance regularly โ€” defined as at least once per week. That is up from 48% in 2023 and 21% in 2022. + +The breakdown by tool: +- GitHub Copilot: 41% (individual or employer-provided) +- Cursor: 22% +- JetBrains AI Assistant: 16% +- Amazon CodeWhisperer/Q: 11% +- Codeium/Windsurf: 14% +- Custom/self-hosted (typically via API): 9% + +Usage patterns are more interesting than adoption rates. Among the 73% who use AI coding tools regularly: +- 44% describe their use as "can't work without it now" +- 38% describe it as "useful but I could work without it easily" +- 18% are "trying to reduce usage" + +That 18% figure is new this year. In 2023, almost no respondents described themselves as trying to reduce AI tool usage. The shift suggests that an early wave of enthusiastic adoption is producing a correction among some users who find the tools changing their work in ways they don't like. Open comments in this category frequently mention concerns about code quality, loss of deep focus on problems, and a feeling of not understanding code they have written. + +## Remote Work in 2024 + +The return-to-office trend has had a limited effect on software developers compared to other knowledge workers. Among our respondents: +- 51% work fully remotely +- 31% work in a hybrid arrangement (typically 2-3 days in office) +- 18% work primarily in-office + +For 2023, the equivalent numbers were 54%, 29%, and 17% โ€” a modest but real shift toward more in-office time, but far from the reversal that many RTO mandates were aiming for. The most common pattern we hear from respondents at companies with mandatory RTO policies is compliance for the minimum required days combined with active job searching for remote-first roles. + +Salary data shows that fully remote roles still command a premium: median reported salary for fully remote roles (among US-based respondents) was $142,000, compared to $134,000 for hybrid and $129,000 for in-office roles. The premium has narrowed from 2022 when remote roles commanded a larger differential, but it persists. + +## Salary Data + +Median reported total compensation by experience (US respondents only, n=1,042): +- 0-2 years experience: $87,000 +- 3-5 years: $122,000 +- 6-10 years: $153,000 +- 11-15 years: $174,000 +- 16+ years: $181,000 + +These figures are self-reported and have not been independently verified. They align broadly with data from other sources, with the caveat that TechPulse's audience skews toward technically ambitious developers who may earn above-median salaries. + +Geographic variance remains enormous. Median compensation in the San Francisco Bay Area for respondents with 6-10 years of experience was $218,000. For equivalent experience in the UK, โ‚ฌ108,000 (approximately $135,000). In Germany, โ‚ฌ95,000. + +## Tooling Preferences + +The text editor and IDE landscape has shifted meaningfully. VS Code remains dominant but is losing ground to AI-native editors: +- VS Code: 48% primary editor (down from 58% in 2023) +- Cursor: 21% (up from 6%) +- JetBrains family: 22% +- Neovim: 7% +- Other: 2% + +Cursor's growth is extraordinary. It has gone from a niche tool to the second most popular primary editor in our survey in a single year. Its adoption appears to be driven primarily by respondents switching from VS Code who want tighter AI integration. + +For build and runtime tooling, the Docker adoption plateau is real: 74% of respondents use Docker regularly, flat versus 2023. Kubernetes usage has declined slightly to 38% of respondents, down from 41%. The platforms taking share are Railway, Fly.io, and direct cloud managed services โ€” respondents are opting for managed solutions rather than self-managed Kubernetes. + +## Burnout + +Thirty-eight percent of respondents describe themselves as experiencing significant burnout in the past 12 months. Thirty-one percent describe burnout as a persistent feature of their work. These numbers are significantly higher than in our 2022 and 2023 surveys (35% and 37% respectively reporting significant burnout), suggesting that the trend is moving in the wrong direction. + +Open responses on burnout cluster around several themes: understaffing amid hiring freezes, pressure to use AI tools without adequate training or time to adapt, meeting load, on-call responsibilities, and a general sense that the pace of change in the field has become unsustainable. + +The finding we find most concerning: among respondents who describe high burnout, 52% are actively looking for a new job or planning to within the next 12 months. The industries and companies most exposed to talent attrition from burnout are those with aggressive RTO policies and highest AI adoption pressure. + +--- + +*Full methodology and data tables are available to TechPulse subscribers. The survey was administered online; respondents were recruited through TechPulse's newsletter, social channels, and partner communities. Results are weighted for company size and geography.* diff --git a/sample-sites/techpulse/posts/2025-01-22-anthropic-o3.md b/sample-sites/techpulse/posts/2025-01-22-anthropic-o3.md new file mode 100644 index 0000000..f705f45 --- /dev/null +++ b/sample-sites/techpulse/posts/2025-01-22-anthropic-o3.md @@ -0,0 +1,55 @@ +--- +title: "Chain-of-Thought Models Change Everything โ€” But Not in the Way You Think" +created: 2025-01-22 09:30 +author: Raj Patel +keywords: reasoning models, chain-of-thought, o1, enterprise AI, LLM limitations, AI reasoning +description: The new generation of reasoning models that think before answering have changed what AI can do. But the change is more specific โ€” and the limitations more persistent โ€” than the coverage suggests. +--- + +The AI industry's coverage of chain-of-thought reasoning models has settled into a predictable pattern: each new release produces a wave of breathless coverage about benchmark scores, followed by a wave of critical coverage pointing out that the benchmarks are gamed, followed by both camps missing the most interesting question, which is: what do these models actually change, in practice, for the people building with them? + +I have spent the past several months talking to enterprise AI teams, individual developers, and AI researchers about their experience with reasoning models โ€” the class of models, exemplified by OpenAI's o1 series and its successors, that spend additional compute "thinking" through problems before producing an output. The picture is genuinely interesting and considerably more nuanced than either the bullish or bearish coverage suggests. + +## What Changed + +The core change is real and important. Prior generation language models โ€” GPT-4, Claude 3, the mid-2024 vintage of large models โ€” produce outputs by processing a prompt and generating a response token by token in a single pass. They are good at tasks that can be solved by pattern matching over their training distribution: writing code that follows common patterns, summarising documents, answering factual questions, translating text. + +They are structurally weak at tasks that require sustained multi-step reasoning โ€” problems where getting the right answer requires holding multiple sub-problems in working memory, checking intermediate conclusions, and revising when those conclusions turn out to be wrong. Mathematical reasoning, complex debugging, multi-constraint optimisation, and formal logic tasks all fall into this category. + +Chain-of-thought reasoning models address this limitation by generating explicit reasoning steps before producing a final answer. The model, in effect, writes a scratchpad of thinking that it then uses to produce a more reliable answer. The "thinking" is itself a generated sequence, which means it can be long, exploratory, and self-correcting in ways that a single-pass generation cannot be. + +The empirical improvement on reasoning tasks is real and substantial. Mathematical benchmark scores, coding competition scores, and formal reasoning task scores all improve significantly with chain-of-thought models. This is not benchmark gaming in the crude sense โ€” the improvements generalise to novel problems of the same type. + +## Where Enterprise Adoption Has Gained Traction + +I spoke with AI leads at seventeen enterprises across financial services, healthcare, software development, and professional services. The common thread in successful deployments is this: chain-of-thought models are making a real difference in tasks that require complex, auditable reasoning, and they are changing little in tasks that don't. + +**Code review and debugging** is the clearest success story. Multiple engineering teams reported that chain-of-thought models are meaningfully better at identifying subtle bugs, understanding complex control flow, and explaining why code is wrong in ways that help developers learn. One senior engineering manager described it as "finally getting a code review from someone who actually thinks it through rather than just pattern-matching on what they've seen before." The caveat: the thinking time means latency is higher, which matters for interactive use but less for asynchronous review workflows. + +**Legal document analysis** in financial services is another genuine success. Reasoning models can work through complex contract logic, identify dependencies between clauses, and flag conflicts that earlier models missed. The combination of reasoning capability and the ability to cite specific text makes them useful for audit-trail purposes in regulated industries. + +**Complex data analysis tasks** โ€” not simple aggregations but multi-step analytical reasoning โ€” are improving. "If I ask it to figure out why our conversion rate dropped last quarter, it can actually work through the possible explanations systematically rather than just listing things that might affect conversion," one data analyst told us. + +## Where They Still Fail + +The failure modes of reasoning models are different from the failure modes of their predecessors but no less real. + +**Reasoning models can reason very confidently toward wrong answers.** The "thinking" process generates internal consistency, but internal consistency is not the same as correctness. I have seen reasoning models produce elaborate, coherent explanations for conclusions that were factually wrong, complete with carefully structured arguments that would require domain expertise to identify as incorrect. This is, in some ways, more dangerous than an older model that produces a wrong answer in an obviously uncertain way. + +**Long-horizon task completion remains elusive.** The improvements in reasoning capability apply within a bounded context: given a well-defined problem, reasoning models are better at finding the answer. They are not significantly better at managing complex projects over time, maintaining consistency across long workflows, or autonomously completing tasks that require adapting to unexpected situations. The vision of AI agents that can work on problems for hours or days remains largely unrealised despite the capability improvements. + +**Domain-specific knowledge limitations persist.** Chain-of-thought reasoning improves formal reasoning but does not substitute for domain knowledge. A reasoning model asked to analyse a clinical trial design will reason more carefully but will still make errors that a domain expert would not, because its underlying knowledge of clinical research methodology is imperfect. Reasoning models are better advisors in areas where careful thinking matters; they are not reliable substitutes for domain expertise. + +**Cost and latency are real constraints.** Reasoning models consume significantly more tokens than standard models, because the thinking process itself generates output that must be processed. API costs for reasoning-heavy tasks can be 5-10x higher than equivalent tasks on standard models. For some high-value tasks this is obviously worthwhile. For high-volume, latency-sensitive applications, it changes the economics significantly. + +## The Pattern That Matters + +The enterprise teams making the best use of reasoning models have converged on a pattern: use them for decisions, not for generation. Standard models are still excellent for generating content โ€” writing emails, summarising documents, producing code scaffolding. Reasoning models are worth their cost for the decision-making steps: reviewing that code, analysing that document for specific logical issues, working through a complex technical question. + +The mistake made by teams that have been disappointed with reasoning models is using them as a drop-in replacement for standard models in generation tasks, where the reasoning capability provides little benefit and the cost and latency increase is pure overhead. + +Chain-of-thought models have changed something real about what AI can do. They have not changed the fundamental challenge of deploying AI in production: knowing precisely what the system can and cannot do reliably, and designing your workflow so that the unreliable parts have appropriate human oversight. + +--- + +*Raj Patel spoke with AI engineering teams at seventeen enterprise companies between October 2024 and January 2025. Companies are not named; they requested anonymity as a condition of participation.* diff --git a/sample-sites/techpulse/posts/2025-03-10-platform-engineering.md b/sample-sites/techpulse/posts/2025-03-10-platform-engineering.md new file mode 100644 index 0000000..e6d6f30 --- /dev/null +++ b/sample-sites/techpulse/posts/2025-03-10-platform-engineering.md @@ -0,0 +1,69 @@ +--- +title: "Platform Engineering Is the New DevOps โ€” And That's Both Good and Bad" +created: 2025-03-10 14:00 +author: Maya Osei +keywords: platform engineering, DevOps, internal developer platforms, CNCF, golden paths, developer experience +description: Platform engineering has become the dominant framework for thinking about internal developer infrastructure. A look at whether it is solving the right problems and what the CNCF data says. +--- + +Every few years, the software industry invents a new term for the cluster of practices around making developers productive in complex organisations. First there was "ops." Then DevOps. Then SRE. Now platform engineering. The question worth asking is whether the new label represents genuine progress in how we think about the problem, or whether it is primarily branding that allows the same arguments to be relitigated with a fresh vocabulary. + +Having spent several months reviewing the CNCF's 2025 platform engineering report, talking to platform teams at companies of varying sizes, and reading the academic and practitioner literature, my answer is: it's some of both, and the difference matters. + +## What Platform Engineering Actually Means + +The CNCF's Platform Engineering Maturity Model, published in 2023 and updated in 2025, defines a platform as "a foundation of self-service APIs, tools, services, knowledge, and support that are arranged as a compelling internal product." Platform engineering is the practice of building and maintaining that foundation. + +The key word in that definition is "product." Platform engineering, as a discipline, differs from traditional infrastructure or DevOps work in its explicit adoption of product thinking: the internal customers of the platform are treated as users whose experience matters, whose needs must be understood through user research and feedback loops, and whose adoption should be measured and optimised. + +This framing comes with a specific set of practices: platform roadmaps that are driven by developer needs rather than just infrastructure requirements, developer experience (DevEx) metrics, golden paths that offer a recommended way to do common tasks without mandating it, and internal marketing for platform capabilities. Many organisations have infrastructure teams that do not think this way; the claim of platform engineering is that this thinking produces better outcomes. + +## What the CNCF Data Shows + +The CNCF surveyed 1,400 organisations for their 2025 report. The headline finding: 71% of organisations with more than 500 engineers have either implemented an internal developer platform or are actively building one, up from 55% in 2023. + +The outcomes data is more interesting than the adoption data. Organisations that scored highly on the CNCF's platform maturity model reported: +- 34% reduction in time from code commit to production deployment +- 28% reduction in developer onboarding time for new team members +- 19% reduction in security incidents related to misconfiguration +- Higher scores on internal developer satisfaction surveys + +These numbers look compelling, but they come with a significant caveat: the organisations that have invested heavily in platform engineering are also the organisations that invest heavily in engineering infrastructure in general. The correlation between platform maturity and deployment efficiency may partly reflect underlying investment levels rather than a causal effect of the platform engineering approach specifically. + +## The Problem Platform Engineering Was Built For + +To assess whether platform engineering is solving the right problems, you need to understand what it is responding to. The problem is genuine: as organisations adopt microservices, containerisation, multiple cloud environments, and complex CI/CD pipelines, the cognitive load on application developers has increased dramatically. + +A developer who needs to ship a feature must now navigate: Kubernetes configuration, service mesh settings, IAM policies, observability instrumentation, security scanning requirements, deployment pipeline configuration, and an ever-growing stack of infrastructure tooling. Each of these things exists for a good reason, but in aggregate they have created a situation where many developers spend more time wrestling with infrastructure than writing application code. + +Platform engineering's answer is the "golden path" โ€” a supported, opinionated way of doing common infrastructure tasks that abstracts the complexity without removing it. Instead of every developer team reinventing CI/CD pipelines, there is a platform-provided pipeline template. Instead of every team figuring out Kubernetes manifests, there is a platform-provided deployment abstraction. + +The golden path metaphor is well-chosen: it is a recommendation, not a mandate. Teams that need to deviate can deviate; they just lose the platform team's support when they do. This is more functional than either "everyone figures it out themselves" or "everyone must use the standard approach regardless of whether it fits their needs." + +## The Problems Platform Engineering Creates + +The problems with platform engineering are less often discussed because they tend to emerge after the implementation phase, when the platform team has already been staffed and positioned as a success. + +**Platform teams become bottlenecks.** When infrastructure decisions require going through the platform team, the platform team must prioritise developer requests. If the platform team is understaffed (common) or slow-moving (also common), developers wait. The same dynamic that DevOps was invented to address โ€” developers waiting for ops to provision infrastructure โ€” can re-emerge if platform teams are not very careful about their operating model. + +**Golden paths become golden cages.** Over time, platform teams tend to optimise their golden paths for the average case, which means they work well for common use cases and badly for edge cases. Teams with unusual requirements โ€” high-performance computing workloads, specialised security requirements, novel architectures โ€” find themselves fighting the platform rather than being helped by it. The cognitive overhead shifts from "figuring out Kubernetes" to "figuring out how to do what I need within the platform's assumptions." + +**Platform engineering replicates without redistributing complexity.** The platform team absorbs infrastructure complexity so application teams don't have to face it directly. But the complexity does not go away โ€” it concentrates in the platform. This is often the right tradeoff. But it means that platform team burnout and turnover is extremely costly, and that organisations are creating a new class of institutional knowledge that is hard to replace. + +## What The Most Successful Implementations Look Like + +The platform teams we interviewed that had the most developer satisfaction and the best outcomes shared several characteristics: + +They measured developer experience systematically, using DORA metrics and DevEx surveys, and used that data to prioritise platform work. They did not assume they knew what developers needed โ€” they asked, regularly. + +They adopted a product development model including a backlog, regular prioritisation, and clear roadmaps, with application developers invited to contribute to priority setting. + +They treated the platform as optional for edge cases, building escape hatches and documenting them. This reduced resistance to the platform and meant developers only fought the golden path when they had genuine reasons to. + +They kept the platform team small and focused on the highest-leverage infrastructure work, resisting the tendency to expand platform scope until scope expansion was clearly justified by developer demand. + +Platform engineering, done well, is genuinely valuable. Done poorly, it is DevOps complexity with a product manager and a roadmap. + +--- + +*Maya Osei covers startups, funding, and the business of developer tools at TechPulse.* diff --git a/sample-sites/techpulse/posts/2025-04-28-open-source-ai-models.md b/sample-sites/techpulse/posts/2025-04-28-open-source-ai-models.md new file mode 100644 index 0000000..65f91f2 --- /dev/null +++ b/sample-sites/techpulse/posts/2025-04-28-open-source-ai-models.md @@ -0,0 +1,67 @@ +--- +title: "Open Source AI Models in 2025: The Landscape Is More Complex Than It Seems" +created: 2025-04-28 11:00 +author: Raj Patel +keywords: open source AI, Llama 3, Mistral, Gemma, open weights, AI licensing, Meta AI +description: Llama, Mistral, Gemma โ€” the "open source AI" movement is growing fast. But what does "open" actually mean when applied to large language models, and which models are actually open? +--- + +The phrase "open source AI model" is used everywhere and means almost nothing consistent. When Meta releases Llama 3, they call it open source. When Mistral releases their models, they call them open source. When Google releases Gemma, they call it open source. In each case, "open source" refers to something meaningfully different, and in most cases it refers to something that the Open Source Initiative and the broader open source community would not recognise as open source in the traditional sense. + +This matters for practical reasons โ€” your ability to use, modify, and redistribute a model depends on the actual terms, not the marketing language. It matters for political reasons โ€” if "open source AI" becomes a term that can be claimed by companies that are merely releasing weights under restricted licences, it dilutes the meaning of open source in ways that will have long-term consequences for the ecosystem. And it matters for philosophical reasons โ€” the debate about what openness means for AI models is substantively different from the debate about what openness means for traditional software, because the artefacts involved are different. + +## What "Open" Can Mean for an AI Model + +Traditional open source software requires, at minimum, that the source code be available and that it can be freely used, modified, and redistributed. The Open Source Definition, maintained by the OSI, has specific criteria. Most "open source" AI models fail these criteria in multiple ways. + +For an AI model, the meaningful components that could be "open" include: + +**Weights** โ€” the numerical parameters that define the model's behaviour after training. Releasing weights allows anyone to run the model and fine-tune it, but without anything else, it is analogous to releasing a compiled binary without source code. + +**Training code** โ€” the code used to train the model, including architecture definitions and training procedures. This is analogous to source code in traditional software. + +**Training data** โ€” the data the model was trained on. This is arguably the most important factor in a model's capabilities and alignment, and the most important thing that is almost never released. + +**Evaluation code and data** โ€” the benchmarks and test sets used to evaluate the model's capabilities. Needed to independently verify capability claims. + +Most "open" AI models release only weights, and often with restrictive licences that prohibit commercial use above a certain scale, require attribution, prohibit certain use cases, or retain the right to revoke the licence. This is not open source in any traditional sense. + +## The Models and What They Actually Release + +**Meta Llama 3** (and the Llama family generally) releases weights under a custom "Meta Llama 3 Community License." The licence allows commercial use but prohibits using Llama to train other large language models (a significant restriction), requires attribution, and prohibits use by entities with more than 700 million monthly active users without a special agreement. Training code is partially available. Training data is not released. + +**Mistral** releases weights for several models under Apache 2.0, which is the most genuinely open licence in the "open" AI model space. Apache 2.0 allows commercial use, modification, and redistribution without restrictions beyond attribution. Mistral does not release training code or training data for its flagship models. Their "open weights" language is more honest than "open source." + +**Google Gemma** uses a custom licence that allows commercial use but prohibits certain applications (explicitly: use in weapons development, surveillance, and certain high-risk medical applications) and restricts redistribution in ways that are not compatible with OSI open source criteria. Training data and training code are not released. + +**Falcon** from the Technology Innovation Institute releases weights under Apache 2.0 for most model sizes, making it one of the more genuinely open options for weights. Like other models, training data is not released. + +**BLOOM** from BigScience is the closest to a genuinely open model โ€” it was trained using a diverse coalition of researchers, the training data (ROOTS) is documented and partially available, and the model is available under a licence that is OSI-compliant in spirit if not letter. + +## The Training Data Problem + +The deepest issue in open source AI models is training data. A model's capabilities, biases, and failure modes are substantially determined by what it was trained on. Without access to training data, you cannot truly audit a model's behaviour, cannot understand why it fails in certain ways, and cannot replicate the training to produce a model with different properties. + +There are legitimate reasons why training data is not released. Much of the text used to train large language models comes from the web and includes copyrighted material โ€” releasing the training data would create enormous copyright exposure. Personal data collected in training sets raises privacy concerns. The compute cost of reproducing a training run from data is prohibitive for most actors. + +These are real constraints, not excuses. But they mean that the most important component for understanding what a model is and why it behaves as it does is, in practice, unavailable. This is a fundamental limitation on the openness of current AI models that is unlikely to be resolved in the near term. + +## Commercial Use Restrictions and Their Implications + +The Llama family's restriction on using its weights to train other large language models is a significant practical constraint that is easy to miss in the licence terms. It means that the Llama models, despite being widely described as "open source," cannot be used to produce derivative foundation models. You can fine-tune Llama for a specific task; you cannot use Llama as the initialisation point for a new pretrained model. + +This restriction protects Meta's competitive position โ€” they do not want to train a model that then gets used to build a competitor โ€” while allowing the application ecosystem to develop. It is a commercially rational choice. It is not consistent with the open source principle that anyone can use open source software as the basis for any project, including a competitive one. + +## The Case for Releasing Weights Anyway + +None of this is an argument that releasing weights is not valuable. It is. Weights-only releases have enabled enormous amounts of useful research, have allowed fine-tuning for specialised domains, have created an ecosystem of tools and applications, and have provided a practical alternative to API-only access for organisations with privacy requirements or latency constraints. + +The argument is specifically about terminology. Calling these releases "open source" obscures the real distinctions between what is genuinely open and what is open in a more limited marketing sense. Those distinctions matter for developers making architectural decisions, for researchers studying AI, and for the policy conversations about AI regulation that increasingly hinge on what "open" means. + +The OSI's ongoing work to define "Open Source AI" โ€” a formal definition that extends their existing principles to AI systems โ€” is an important contribution to this conversation. Their current draft requires, at minimum, that training data be documented and described (not necessarily released), that training code be released, and that weights be released under an OSI-approved licence. By these criteria, almost no current major AI model qualifies as open source. + +That gap between the marketing language and the formal definition deserves more attention than it gets. + +--- + +*Raj Patel has been following the open source AI ecosystem since the Llama 1 release. He has no financial relationship with any of the companies mentioned.* diff --git a/sample-sites/techpulse/posts/2025-06-15-kubernetes-fatigue.md b/sample-sites/techpulse/posts/2025-06-15-kubernetes-fatigue.md new file mode 100644 index 0000000..7a414d1 --- /dev/null +++ b/sample-sites/techpulse/posts/2025-06-15-kubernetes-fatigue.md @@ -0,0 +1,80 @@ +--- +title: "Kubernetes Fatigue Is Real โ€” Here's What Teams Are Doing Instead" +created: 2025-06-15 09:00 +author: Clara Winthorpe +keywords: Kubernetes, k8s, platform alternatives, Fly.io, Railway, managed services, infrastructure fatigue +description: Our survey of 200 engineering teams finds growing Kubernetes fatigue. We look at who is leaving, who is staying, and what the alternatives actually look like in practice. +--- + +The Kubernetes enthusiasm that dominated infrastructure conversations from 2018 through 2022 has given way to something more complicated: a widespread recognition that Kubernetes is extraordinary infrastructure for a specific set of problems, and an unfortunate default solution for many problems it is not well-suited for. + +Over eight weeks, TechPulse surveyed 200 engineering teams about their infrastructure choices, their experience with Kubernetes, and what they have changed or considered changing in the past 18 months. The findings suggest a market in transition โ€” not a Kubernetes collapse, but a significant reassessment. + +## The Survey Findings + +Of the 200 teams surveyed, 142 were using Kubernetes at the time we spoke with them. Of those: +- 31% described themselves as "fully satisfied" with Kubernetes for their workloads +- 44% described it as "working well but resource-intensive to maintain" +- 18% described it as "more trouble than it's worth for our scale" +- 7% were "actively looking to migrate away from it" + +Among the 58 teams not using Kubernetes: +- 22 had migrated off Kubernetes in the past two years +- 19 had evaluated Kubernetes and decided not to adopt it +- 17 had never seriously considered it + +## Who Is Going Back to VMs + +The teams that have migrated away from Kubernetes share a recognisable profile: they are typically under 30 engineers, running fewer than ten services, and they adopted Kubernetes because it was the industry default rather than because their workloads specifically required it. + +"We had three microservices and six engineers and we were running Kubernetes because that's what you did," the CTO of a 20-person SaaS company told us. "We spent the equivalent of one full-time engineer's time just managing the cluster. We moved to managed VMs and a simpler deployment process and we have not looked back." + +Several teams have returned to virtual machines, either managed directly through cloud providers or through platforms that provide a higher level of abstraction over VMs. The appeal is straightforwardness: a VM does what you tell it to, has a clear resource model, and does not require understanding pod scheduling, ingress controllers, persistent volume claims, and a dozen other Kubernetes-specific concepts. + +The cost argument is also real. Kubernetes clusters have a minimum overhead โ€” the control plane, the node agents, the namespace overhead โ€” that adds up for small workloads. A small application running on a properly-sized VM is often significantly cheaper than the same application running in a Kubernetes cluster, even before you account for the engineering time. + +## Managed Services: The Middle Ground + +The most common "instead of Kubernetes" choice in our survey was not VMs but managed services โ€” giving responsibility for the infrastructure to a cloud provider or a specialist platform and focusing on application code. + +AWS Fargate, Google Cloud Run, and Azure Container Apps all provide container running services without requiring Kubernetes knowledge. The tradeoffs are loss of control and increased per-unit cost, but for many teams the operational simplification justifies both. + +Among smaller and developer-first companies, Fly.io and Railway have attracted significant attention and, in some cases, meaningful migration away from both Kubernetes and AWS-native services. + +## Fly.io: What the Hype Is About + +Fly.io has built an infrastructure layer that runs containers close to users at a global network of data centres, with a developer experience that is genuinely simple compared to Kubernetes. Deployments happen from the command line with `fly deploy`. Machines start in seconds. Pricing is straightforward and usage-based. + +The teams we spoke with that have moved to Fly.io consistently described the same thing: a dramatic reduction in infrastructure cognitive load. "I stopped spending Sunday evenings worrying about the cluster," one engineering lead said. "The infrastructure just runs." + +The limitations are real. Fly.io gives you less control than Kubernetes โ€” you cannot bring arbitrary infrastructure, the networking model is fixed, and the platform is still maturing. Several teams we spoke with noted that they had hit edge cases where Fly.io's abstraction made it difficult to do things that Kubernetes made possible, though complex. + +Fly.io also had several well-publicised reliability incidents in 2023 and early 2024 that caused teams evaluating it to pause. The company has invested significantly in reliability since then, and the teams currently running production workloads on it describe good availability. But the incidents created a trust deficit that is still being rebuilt. + +## Railway: The Simplest Option + +Railway occupies an even simpler position than Fly.io. It is closer to a PaaS โ€” you give it a GitHub repository and environment variables, and it figures out how to run your code. There is almost no infrastructure configuration. The target user is a developer who wants to run a backend service without any ops work. + +Among the teams we spoke with, Railway is primarily used for internal tools, side projects, and small production services where engineering simplicity is the primary goal and workload characteristics are predictable and modest. It is not being seriously evaluated for large-scale, complex production workloads. It is not trying to be. + +## The Teams Staying With Kubernetes โ€” And Why + +The 31% of Kubernetes teams that described themselves as fully satisfied are worth understanding. What do they have in common? + +They are large enough to justify dedicated platform engineering. Every fully-satisfied team we spoke with had at least two engineers whose primary focus was platform and infrastructure. This creates a different experience: Kubernetes is complicated to maintain, but if someone's job is maintaining it, the complexity becomes manageable. + +Their workloads genuinely benefit from Kubernetes' capabilities. The teams most satisfied with Kubernetes are running workloads with heterogeneous resource requirements, complex networking, stateful services that benefit from Kubernetes-native storage, or compliance requirements that benefit from Kubernetes' audit and access control capabilities. + +They have invested in abstractions above raw Kubernetes. Every satisfied Kubernetes team we spoke with had built internal tooling or adopted a platform layer โ€” Helm charts, Backstage, Crossplane, or an internal developer platform โ€” that abstracted the Kubernetes complexity for application developers. They were, in effect, doing platform engineering. + +## The Real Lesson + +The lesson of the Kubernetes fatigue phenomenon is not that Kubernetes is a bad technology. It is an extraordinary piece of software that has enabled an entire generation of complex distributed systems. The lesson is about fit: Kubernetes is the right tool for a specific class of problems, and it was adopted as the universal solution for all problems, which it is not. + +The teams most satisfied with Kubernetes know exactly why they are using it. The teams most frustrated with it are often teams that adopted it because everyone else was doing so, without asking whether the tradeoffs made sense for their scale, team size, and workload characteristics. + +The infrastructure market is slowly recalibrating toward fit-for-purpose choices. That recalibration is healthy. + +--- + +*Clara Winthorpe covers infrastructure, open source, and DevOps at TechPulse. She surveyed 200 engineering teams between March and May 2025.* diff --git a/sample-sites/techpulse/posts/2025-07-22-vc-funding-2025.md b/sample-sites/techpulse/posts/2025-07-22-vc-funding-2025.md new file mode 100644 index 0000000..c2f4899 --- /dev/null +++ b/sample-sites/techpulse/posts/2025-07-22-vc-funding-2025.md @@ -0,0 +1,66 @@ +--- +title: "Tech Funding in 2025: The AI Bubble vs. The Infrastructure Boom" +created: 2025-07-22 10:30 +author: Maya Osei +keywords: venture capital 2025, AI funding, infrastructure investment, IPO market, tech funding trends +description: H1 2025 funding data shows a bifurcated market โ€” AI application layer funding has cooled while infrastructure investment continues to grow. What the LP perspective reveals. +--- + +Two narratives are simultaneously true about technology venture capital in the first half of 2025: the AI funding boom is moderating, and infrastructure investment is accelerating. These trends are related, and understanding the relationship between them explains a lot about where the technology industry is heading. + +Total venture investment in technology companies in H1 2025 was approximately $89 billion globally, according to PitchBook data. That is 14% below H1 2024's total of $103 billion, but still far above pre-2020 levels. The decline is concentrated in AI application companies; AI infrastructure and developer tools have continued to attract capital at elevated rates. + +## The Application Layer Correction + +The correction in AI application company valuations has been building since late 2024. The proximate cause is the same one that has historically corrected inflated software valuations: enterprise customers are taking longer to convert from pilots to paying contracts, and the ARR metrics that were used to justify high valuations in 2023-2024 have not been sustained as customers churn out of first-year contracts at unexpectedly high rates. + +The median AI application company raised its last round at approximately 25x ARR in 2023. Those companies are now struggling to raise follow-on rounds at those multiples, because investors are applying 2024's harder-earned scepticism to ARR quality. The companies in trouble are those with: + +- High customer concentration (top three customers representing more than 50% of ARR) +- Churn rates above 15% annually on an ARR basis +- Products that are wrappers over foundation models without genuine differentiation +- Burn multiples above 2.0x (spending $2 or more to generate each $1 of ARR) + +By these criteria, a large fraction of the AI application cohort is facing a difficult fundraising environment. Several well-known companies that raised at unicorn valuations in 2023 have not yet announced follow-on rounds; the absence of news, in this environment, is informative. + +## Infrastructure Investment Continues + +The infrastructure layer tells a different story. Spending on AI compute, networking, and datacenter capacity remains extraordinary. NVIDIA's revenue has continued to grow, and the cloud providers' capital expenditure on GPU clusters has not shown signs of moderating. The constraint on AI infrastructure investment is not demand โ€” enterprise demand for AI compute remains robust โ€” but the rate at which new hardware can be manufactured and deployed. + +The venture investment picture at the infrastructure layer reflects this dynamic. Companies building AI inference infrastructure, specialised AI hardware, and the data pipelines and MLOps tooling that enterprises need to operate AI systems in production are raising rounds at healthy valuations with relatively less valuation compression than the application layer. + +The pattern is recognisable from the cloud computing buildout of the 2010s: the infrastructure layer captured durable, growing revenue before the application layer had figured out its business models. AWS became an enormous, profitable business while hundreds of cloud-native application companies struggled with unit economics. + +## The IPO Market: Still Frozen, But Thawing + +The technology IPO market has been largely frozen since 2022's correction. 2023 and 2024 saw very few significant tech IPOs, as companies with access to private capital preferred to stay private rather than face the scrutiny and short-term pressure of public markets. + +In H1 2025, we saw the first tentative signs of thaw. Several mid-sized technology companies filed for IPO or completed listings, testing whether public market investors had re-calibrated their expectations from the froth of 2021. Early results are mixed. Companies with clear profitability paths and strong unit economics have been received reasonably well. Companies without have faced a chilly reception. + +The companies most likely to successfully IPO in the next 12-18 months are infrastructure and developer tools companies with strong recurring revenue and clear paths to profitability. Application-layer AI companies are unlikely to have attractive IPO conditions until the churn and ARR quality questions in the sector are more clearly resolved. + +## The LP Perspective + +The most revealing conversations I had for this piece were with limited partners โ€” the endowments, pension funds, and family offices that fund venture capital funds. LPs have historically been the most accurate leading indicators of VC market conditions, because they are investors in investors: their behaviour predicts what VCs will be able to deploy. + +The pattern among LPs in 2025 is a meaningful reduction in new commitments to early-stage generalist venture funds, offset by continued strong interest in growth-stage infrastructure-focused funds and specialised AI infrastructure funds. "We got burned by the 2021 vintage," one endowment manager told me. "Not catastrophically โ€” but the returns are going to be mediocre for a generation of funds that looked like obvious winners in 2022. We're being more selective." + +The practical implication: the pool of capital available for early-stage AI application company fundraising has contracted more significantly than the headline numbers suggest, because LP caution flows through to new fund formation, which then flows through to early-stage investment 12-18 months later. + +## Which Sectors Are Actually Drying Up + +Beyond the AI application layer, several other categories are experiencing meaningful funding contractions: + +**Consumer fintech** โ€” the category that produced a wave of neobanks, BNPL services, and financial apps in 2019-2022 โ€” has been in secular decline since interest rates rose. Business models that depended on low-cost capital and high consumer spending have proven fragile. + +**SaaS without AI features** โ€” pure workflow management, project tracking, and similar tools without meaningful AI integration are struggling to raise at previous multiples. The bar for "what is distinctive about this product" has been reset by the AI-native competitors entering every software category. + +**Web3/crypto-adjacent infrastructure** โ€” after the 2022-2023 shakeout, institutional venture capital has largely abandoned the sector except for the infrastructure layer (custody, compliance tools, institutional trading infrastructure). Consumer-facing crypto products are finding little VC interest. + +**No-code and low-code platforms** โ€” the category that was expected to democratise software development in the early 2020s has been significantly disrupted by AI coding tools, which address the same underlying demand more effectively for the use cases that matter most to enterprise buyers. + +The bifurcated market will resolve, as bifurcated markets always do, either by the infrastructure investment producing revenue that justifies it, or by a broader reassessment. The evidence currently points more toward the former, but the timeline is uncertain. + +--- + +*Maya Osei covers the business of technology and startup funding at TechPulse. Revenue and funding data from PitchBook, CB Insights, and Crunchbase.* diff --git a/sample-sites/techpulse/posts/2025-09-08-deno-v3.md b/sample-sites/techpulse/posts/2025-09-08-deno-v3.md new file mode 100644 index 0000000..34ea234 --- /dev/null +++ b/sample-sites/techpulse/posts/2025-09-08-deno-v3.md @@ -0,0 +1,79 @@ +--- +title: "Deno v3: Is Node.js Compatibility Finally Good Enough?" +created: 2025-09-08 13:00 +author: Clara Winthorpe +keywords: Deno v3, Node.js, npm compatibility, JavaScript runtime, Bun, benchmarks +description: Deno v3 launched with the strongest Node.js compatibility claim the project has ever made. We ran the benchmarks, tested real packages, and talked to teams who have actually migrated. +--- + +When Ryan Dahl unveiled Deno in 2018 as a "do-over" of Node.js, he was explicit about what he was moving away from: npm, the node_modules directory, non-Promise async patterns, and a handful of other decisions that had come to define Node's complexity. Deno would use URLs for imports, browser-compatible APIs, and TypeScript natively. It would be simpler and more secure. + +The problem, which became apparent relatively quickly, was that the JavaScript ecosystem had spent a decade building on npm. Nearly every useful library, framework, and tool was distributed through npm and assumed Node.js internals. Deno's principled stance on clean-room design made it largely incompatible with the ecosystem developers needed. + +Deno's history since 2018 has been the story of gradually reconciling that incompatibility. Deno v2 added the `node:` protocol for Node.js built-in modules and significantly improved npm package compatibility. Deno v3, released in July 2025, makes the strongest compatibility claim in the project's history. + +## What's New in v3 + +Deno v3's headline feature is what the team calls "seamless Node.js compatibility." The specific claims: + +- Full compatibility with the `node:` built-in modules, including `child_process`, `cluster`, `worker_threads`, and other modules that were previously partially supported +- Native npm package installation without an explicit compatibility flag โ€” `deno install` now installs from npm registries by default +- Compatibility with the most popular Node.js frameworks: Express, Fastify, NestJS, and others that depend on Node-specific APIs +- A new `deno run --compat` mode that activates additional Node.js compatibility shims for legacy code + +The team has also improved performance in v3. The V8 version has been updated, the runtime startup time has been reduced, and the built-in TypeScript compilation (Deno compiles TypeScript without a separate build step) has been made significantly faster. + +## Our Benchmark Results + +We ran a suite of benchmarks comparing Deno v3 to Node.js 22 and Bun 1.1 (the latest stable version as of this writing). Tests were run on a dedicated bare-metal server with an Intel Core i9-13900K and 64GB RAM, running Ubuntu 24.04 LTS. + +**HTTP throughput (requests/second, simple JSON response):** +- Bun 1.1: 142,000 req/s +- Node.js 22: 89,000 req/s +- Deno v3: 97,000 req/s + +**Startup time (cold start to first output):** +- Bun 1.1: 12ms +- Deno v3: 31ms +- Node.js 22: 48ms + +**TypeScript compilation of a 50K line codebase:** +- Bun 1.1 (with transpile-only, no type checking): 0.8s +- Deno v3 (with type checking): 4.2s +- Node.js 22 + tsc: 7.1s + +The performance picture is interesting. Bun remains the fastest runtime in most benchmarks, and its lead in HTTP throughput is significant. Deno v3 has closed the gap with Node.js and in several microbenchmarks exceeds it. For real-world web application workloads, the difference between Deno and Node is unlikely to be a deciding factor; the difference between either of them and Bun is more meaningful for latency-sensitive applications. + +Deno's TypeScript handling is a genuine advantage over Node.js's conventional approach (separate TypeScript compilation step via tsc or esbuild) in developer experience terms. Running TypeScript files directly with `deno run` without a build step is convenient and works well in v3. + +## npm Compatibility: How Well Does It Actually Work + +The honest answer to "is Node.js compatibility finally good enough?" is: for most packages, yes; for some important ones, not yet. + +We tested 50 npm packages across different categories. 43 installed and ran without errors under Deno v3. Seven had issues ranging from minor (deprecation warnings about Node-specific patterns) to significant (package depends on native C++ extensions that require a Node.js runtime). + +The category that remains problematic is native modules โ€” npm packages that include compiled C++ extensions. These packages use Node's N-API or NAN to interface with native code, and while Deno has implemented N-API support, it is incomplete. Several popular packages in the cryptography, image processing, and database connector categories use native modules and do not work fully under Deno v3. + +For pure JavaScript/TypeScript packages and packages that use only WASM for native performance, compatibility is excellent. For packages with native C++ dependencies, Deno's story is still incomplete. + +## Real Migration Stories + +We spoke with four teams that had migrated production workloads from Node.js to Deno in the past year. + +A developer tools startup migrated their CLI tooling from Node.js to Deno v3. Their assessment: "We rewrote the parts that used native modules in pure TypeScript and WebAssembly. For everything else, the migration was relatively smooth. We are not significantly faster than we were on Node, but the developer experience is better โ€” first-class TypeScript, better security defaults, and a simpler dependency management story." + +A media company migrated their content delivery backend, a Node.js Express application, to Deno v3. They used Deno's `--compat` flag and described a three-day migration process. "Express runs fine on Deno v3. We hit a few edge cases with middleware that used Node-specific stream APIs, but the Deno team was responsive on GitHub and we had working workarounds quickly. Our throughput is slightly better โ€” maybe 10% on our workloads โ€” but the main win is simpler deployments with Deno's built-in tooling." + +A fintech startup evaluated migrating their Node.js backend but decided against it. The decision came down to native modules: their PostgreSQL connection pooling library used a native module for performance, and the Deno N-API implementation was not stable enough for their production requirements. + +## The Bun Comparison + +Bun's competitive position relative to Deno has not changed significantly with v3. Bun's strengths โ€” exceptional performance, near-complete Node.js compatibility, very fast npm operations โ€” remain its strengths. Deno's strengths โ€” security-first design, first-class TypeScript with full type checking, WebAssembly-native capabilities, the Deno Deploy hosting platform โ€” are different and remain unchanged. + +The two runtimes serve somewhat different audiences. Teams that want the fastest possible Node.js replacement with the minimum migration friction are better served by Bun. Teams that value security boundaries, TypeScript correctness, and are building greenfield projects without heavy npm ecosystem dependencies are better served by Deno. + +Both are genuine improvements on Node.js in their respective domains. Node.js remains the default choice by inertia, ecosystem depth, and the enormous installed base. Whether either alternative achieves significant market share in the runtime market over the next several years depends on whether they can convert that inertia โ€” which is the real challenge. + +--- + +*Benchmark testing conducted on a dedicated server running Ubuntu 24.04 LTS. All runtime versions are as of publication date. Clara Winthorpe covers infrastructure and developer tools at TechPulse.* diff --git a/sample-sites/techpulse/posts/2025-10-14-security-supply-chain.md b/sample-sites/techpulse/posts/2025-10-14-security-supply-chain.md new file mode 100644 index 0000000..a3168a6 --- /dev/null +++ b/sample-sites/techpulse/posts/2025-10-14-security-supply-chain.md @@ -0,0 +1,71 @@ +--- +title: "Software Supply Chain Security in 2025: Progress Report" +created: 2025-10-14 09:00 +author: Raj Patel +keywords: software supply chain security, SBOM, sigstore, GitHub security, dependencies, enterprise security +description: Three years after the xz incident and a decade after SolarWinds, we assess how much enterprise software supply chain security has actually improved. +--- + +The phrase "software supply chain security" entered mainstream awareness after SolarWinds in 2020 and has been a fixture of security conference agendas and government executive orders ever since. In the years since, significant resources have been invested in improving the situation: new tooling, new standards, regulatory pressure, and corporate security programmes specifically targeting the supply chain. + +It is worth asking whether it has worked. + +The honest answer, based on conversations with security researchers, enterprise security teams, and the people building the tooling, is: meaningfully yes in some areas, frustratingly stagnant in others, and still dramatically under-resourced overall. + +## SBOM Adoption: From Mandate to Practice + +The Software Bill of Materials (SBOM) โ€” a machine-readable inventory of the components in a software product โ€” has moved from a security researcher wish list item to a regulatory requirement in several jurisdictions. The US government's Executive Order on cybersecurity in 2021 required federal contractors to provide SBOMs. The EU Cyber Resilience Act, which came into force in phases beginning in 2024, requires SBOMs for products sold into the European market. + +Adoption among enterprises has been driven primarily by these compliance requirements. Our conversations with security teams at large enterprises suggest that SBOM generation is now routine for companies operating in regulated sectors or selling to government customers. The tooling has matured: Syft, CycloneDX, and SPDX toolchains work reliably, integrate with major CI/CD platforms, and produce SBOMs that can be ingested by vulnerability scanning tools. + +The gap is in what organisations do with SBOMs once they have them. Generating an SBOM is the easy part. Ingesting SBOMs from third-party software vendors, maintaining them across complex software ecosystems, and using them for actual risk management decisions is the hard part, and that work is much less mature. + +"We generate SBOMs for everything we ship," one security engineer at a large technology company told us. "But we receive SBOMs from fewer than 10% of our suppliers, and even when we do, we don't have the process to operationalise them into real risk decisions. They exist in a system and nobody looks at them." + +## Sigstore and the Signing Infrastructure + +Sigstore โ€” the open source project providing signing and verification infrastructure for software artefacts โ€” has become genuine critical infrastructure. It is now used to sign virtually all Python packages uploaded to PyPI, the npm registry has begun adopting it for provenance information, and the Go module proxy uses it for module signing. + +The technical architecture is solid: Sigstore uses a public, append-only transparency log (Rekor) that allows anyone to verify that a signing event occurred and when, combined with short-lived signing certificates from Fulcio that tie signatures to verified identities (typically GitHub Actions workflows or similar OIDC-based identities). The result is a system that provides meaningful verification without requiring developers to manage long-lived private keys โ€” a significant usability improvement over PGP-based signing. + +The challenge is adoption at the consumer end. Signing artefacts is now easy; verifying signatures before using them is still a manually-configured step that most developers do not take. Defaulting verification on in package managers โ€” so that installing an unsigned or unverified package requires an explicit opt-in โ€” is the next step that would translate signing infrastructure into routine security practice. npm has begun moving in this direction; others are more cautious. + +## GitHub Dependency Scanning: What It Catches, What It Misses + +GitHub's Dependabot and the code scanning features built on CodeQL have become the most widely-deployed dependency security tooling in the industry, by virtue of being free and integrated into the platform where most open source and a significant fraction of commercial software development happens. + +The effectiveness data is encouraging at the surface level: Dependabot has helped organisations patch millions of vulnerable dependencies. The catch rate for known vulnerabilities with published CVEs is high. + +The limitations are important. Dependabot catches dependency versions with known CVEs. It does not catch malicious packages designed to look like legitimate ones (typosquatting), does not catch compromised legitimate packages (the xz problem), and does not analyse the supply chain of the dependencies themselves โ€” only the direct and transitive dependency versions. + +"Dependabot is extremely good at what it does and that's a real improvement," one security researcher told us. "But the attacks that actually scare me โ€” the patient, sophisticated supply chain compromises โ€” are exactly the ones it doesn't catch, almost by definition." + +## What Enterprises Are Actually Doing + +We spoke with security teams at twelve large enterprises across financial services, technology, and healthcare. The practices they described, in aggregate, suggest a sector that is significantly better than it was in 2020 but still operating below the level that the threat environment warrants. + +**Practices now considered routine** (done by the majority of companies we spoke with): automated dependency scanning in CI/CD pipelines, SBOM generation for shipped software, periodic security reviews of critical open source dependencies, secrets scanning in repositories. + +**Practices being adopted but not yet routine**: SBOM ingestion from suppliers, software composition analysis going beyond CVE scanning to include licence compliance and quality metrics, signing and verification of container images throughout the deployment pipeline. + +**Practices still in early stages**: dependency build reproducibility verification, behavioural analysis of new package versions, automated provenance verification at package install time. + +The remaining gaps centre on the sophistication of the threat model. The tooling that has been deployed broadly addresses the 2015-2019 threat model: known vulnerabilities in known packages. The current threat model includes supply chain attacks against the development and build infrastructure of trusted packages, compromised maintainer accounts, social engineering of tired maintainers (xz), and sophisticated malicious packages designed to evade static analysis. + +## What Would Actually Move the Needle + +The security researchers and practitioners we spoke with converge on several interventions that would have the highest impact: + +**Funded maintenance of critical open source projects.** The xz incident demonstrated that the most dangerous attack vector is not technical but social. Funded, supported maintainers who have colleagues and review processes are significantly harder to compromise than exhausted individuals maintaining projects alone. + +**Default verification in package managers.** Making signature verification mandatory by default โ€” so that installing an unsigned package requires explicit user action โ€” would create a floor of supply chain integrity without requiring developer behaviour change. + +**Build reproducibility at scale.** Reproducible builds โ€” where the same source code and build inputs always produce identical binary outputs โ€” allow independent verification that a distributed binary corresponds to its published source. Tools for achieving reproducibility have improved, but adoption at scale remains incomplete. + +**Faster CVE response infrastructure.** The time between vulnerability discovery and the publication of patches and advisories in machine-readable form remains too long. Streamlining this pipeline would allow the defensive tooling to respond faster. + +The progress in software supply chain security over the past five years is real and should not be minimised. The gap between where we are and where we need to be is also real and should not be minimised. + +--- + +*Raj Patel has covered software security and supply chain issues at TechPulse since 2022. He spoke with twelve enterprise security teams and eight independent security researchers for this article.* diff --git a/sample-sites/techpulse/posts/2025-11-30-ai-agents-enterprise.md b/sample-sites/techpulse/posts/2025-11-30-ai-agents-enterprise.md new file mode 100644 index 0000000..64edcd8 --- /dev/null +++ b/sample-sites/techpulse/posts/2025-11-30-ai-agents-enterprise.md @@ -0,0 +1,75 @@ +--- +title: "AI Agents in the Enterprise: What Actually Works" +created: 2025-11-30 11:00 +author: Maya Osei +keywords: AI agents, enterprise AI, automation, LLM agents, autonomous AI, AI guardrails +description: Case studies from five companies reveal what AI agents are reliably delivering in enterprise settings โ€” and why autonomous decision-making remains out of reach. +--- + +The AI agent narrative has been one of the most persistent stories in enterprise technology for the past two years. Agents โ€” AI systems that can autonomously execute multi-step tasks, use tools, and adapt to unexpected situations โ€” represent the promise of AI that acts rather than just advises. Investors have deployed billions into agent companies. Enterprise technology buyers have run pilots. The results, as of late 2025, are instructive. + +Over three months, TechPulse conducted detailed case study interviews with five companies that have deployed AI agents in production. We also spoke with security, legal, and compliance teams who have been asked to evaluate agent deployments. What we found is a genuine technology making real contributions in specific, bounded use cases, and a gap between that reality and the full autonomy narrative that is wider than most coverage acknowledges. + +## Company One: Financial Services โ€” Document Processing + +A large financial services firm deployed AI agents for initial processing of loan application documents in late 2024. The agent receives a loan application package, extracts structured data from unstructured documents (income statements, employment letters, bank statements), identifies missing documents, and produces a structured summary for human underwriters. + +The results have been positive. Processing time for the document extraction and initial structuring stage has been reduced by approximately 60%. Underwriter time previously spent on document organisation is now spent on actual underwriting decisions. Error rates in data extraction have decreased compared to the manual baseline. + +The key design decision that made this work: the agent operates within a tightly constrained task scope. It extracts and structures data. It does not make lending decisions. It does not send external communications. All outputs are reviewed by a human underwriter before any action is taken. The system failed several times during the pilot โ€” wrong extractions, missed documents, format misinterpretations โ€” but because the human review step was mandatory, none of those failures reached customers. + +"The success comes from keeping the agent in a well-defined box and having a human at the exit of that box," the technology lead told us. "When we tried expanding the scope to include initial underwriting recommendations, the failure rate was unacceptably high and the failures were not always predictable." + +## Company Two: Software Development โ€” Code Review + +An enterprise software company with over 1,000 engineers deployed AI agents for an initial code review pass. The agent reviews pull requests for common issues: potential security vulnerabilities, test coverage gaps, code style violations, and straightforward logic errors. It comments directly on pull requests before human review. + +The outcomes are mixed but net positive. Engineers report that the agent catches approximately 30% of the issues that human reviewers would have caught, which meaningfully reduces the time human reviewers spend on mechanical issues. The agent also catches issues that human reviewers would have missed โ€” it is thorough in a way that humans under time pressure are not. + +The failure mode is false positives. The agent comments on issues that are not actually issues at a rate that engineers find annoying but tolerable. Early versions of the system had a higher false positive rate; prompt engineering and fine-tuning on the company's specific codebase have reduced it to a level that engineers describe as "better than tolerable." + +The limits of the system are clear: it identifies potential issues but the resolution of those issues remains entirely with human engineers. When the agent suggests a fix, engineers review the suggestion carefully and often reject it. The agent's code generation is treated as a starting point, not a trusted output. + +## Company Three: HR โ€” Candidate Screening + +A professional services firm deployed AI agents to help with initial candidate screening for entry-level positions. The agent reviews CVs, identifies candidates that meet basic threshold criteria, and generates a structured assessment of each candidate for human recruiters. + +This deployment has been the most controversial case study. The firm has observed a reduction in screening time per candidate, but they have also had to navigate significant legal and HR concern about AI decision-making in the hiring process. Several jurisdictions have enacted or are considering legislation requiring disclosure when AI is used in hiring decisions. + +The practical adjustment has been to treat the agent's assessment as a search and organisation tool rather than a decision tool. It finds and structures information; it does not recommend hiring or rejection. Human recruiters review every structured assessment before any candidate communication occurs. + +The firm's legal team has been the most sceptical of the deployment. "The AI optimises for patterns in historical data," their legal director noted. "Historical data reflects historical hiring decisions, which have biases. We have invested significant effort in audit frameworks to identify whether the agent is introducing or amplifying bias. We have not found evidence of it, but we have also not had the system in production long enough to have high confidence." + +## Company Four: Customer Support โ€” Tier-One Resolution + +A technology company deployed AI agents to handle initial customer support queries, with the goal of resolving common issues without human intervention and escalating to human agents for complex cases. + +After six months in production, the agent handles 62% of inbound queries without escalation. Customer satisfaction scores for agent-handled queries are lower than for human-handled queries, but within acceptable parameters. Escalation accuracy โ€” the agent's ability to identify which queries need human handling โ€” is the most important metric and has improved significantly from the initial deployment. + +The failure modes are instructive. The agent handles common, well-defined problems (password resets, subscription changes, billing inquiries) very well. It handles novel or ambiguous problems poorly, and it does not reliably recognise when a problem is outside its competence. Early versions of the system would confidently provide incorrect information about product features or policies rather than escalating. This has been addressed through explicit escalation triggers and confidence thresholds, but the company's engineers described ongoing tuning work as "more labour-intensive than expected." + +## Company Five: Legal โ€” Contract Review + +A law firm uses AI agents as a first-pass reviewer for standard commercial contracts. The agent reviews contracts for common issues: missing standard clauses, non-standard terms in key provisions, and potential conflicts with the client's standard positions. + +Lawyers at the firm describe the agent as genuinely useful for speeding up the mechanical review of routine contracts. It does not change how they work on complex, negotiated agreements โ€” it is used for the volume work. + +The guardrails in place are significant: all agent outputs are reviewed by qualified lawyers, outputs are never provided directly to clients, and the firm does not market the AI assistance as part of its service model. "We treat it the way we treat a first-year associate's work," one senior partner said. "Review everything. Trust nothing until you've checked it. Learn to read the failure modes." + +## The Consistent Findings + +Across these five case studies, several consistent findings emerge: + +**Successful deployments are bounded.** Every successful agent deployment we encountered operates within a tightly defined scope with explicit constraints on what the agent can do, what data it can access, and what actions it can take without human approval. + +**Human review is non-negotiable for consequential outputs.** No company we spoke with had removed human review from the path to consequential decisions. The value of agents is in reducing the time humans spend on mechanical aspects of a task, not in removing humans from the loop. + +**Failure modes are not always predictable.** All five deployments experienced failure modes that were not anticipated during the pilot phase. The characteristic of production AI deployment is discovering new failure modes over time, which requires ongoing monitoring and prompt/system adjustment. + +**Autonomous decision-making is where all five companies drew the line.** When we asked each company what they had tried and decided not to deploy, the answers clustered around autonomous decision-making tasks โ€” anything where the agent's output would directly trigger an action without human review. Legal liability, regulatory compliance, and customer trust concerns are cited, but underneath them is a practical concern: no one has confidence in the reliability of autonomous agent decision-making at the level needed for consequential actions. + +The AI agent story is real. It is just a narrower story than the investment narrative suggests. + +--- + +*Maya Osei conducted case study interviews between August and November 2025. Companies are anonymised; descriptions may include minor modifications to protect confidentiality.* diff --git a/sample-sites/techpulse/posts/2025-12-20-developer-predictions-2026.md b/sample-sites/techpulse/posts/2025-12-20-developer-predictions-2026.md new file mode 100644 index 0000000..69bf320 --- /dev/null +++ b/sample-sites/techpulse/posts/2025-12-20-developer-predictions-2026.md @@ -0,0 +1,85 @@ +--- +title: "TechPulse Predictions for 2026: Ten Bets on Developer Technology" +created: 2025-12-20 10:00 +author: Clara Winthorpe +keywords: 2026 predictions, developer technology, AI tools, WebAssembly, open source, programming trends +description: We make ten specific predictions for developer technology in 2026, and look back honestly at how our 2025 predictions fared. +--- + +Every December, TechPulse makes ten specific predictions about the coming year in developer technology. Specific predictions are more useful than vague ones, because specific predictions can be falsified โ€” you can tell at the end of the year whether they were right, partly right, or wrong. The discipline of specificity is also good for thinking: it is easy to say "AI will be important in 2026." It is harder to say something specific, and the hardness is where the thinking happens. + +Before the 2026 predictions, an accounting of our 2025 predictions: + +**2025 retrospective:** + +1. โœ“ *Rust in the Linux kernel will exceed 50,000 lines by end of 2025.* Correct. We put it at approximately 67,000 lines as of December. + +2. โœ— *At least one major AI coding assistant will ship an "offline mode" using a locally-run model.* Wrong. Copilot and Cursor both added local model features, but not the fully offline, enterprise-focused product we predicted. + +3. โœ“ *The term "open source AI" will be formally disputed in a significant policy context.* Correct. The EU AI Act negotiations produced exactly this dispute. + +4. โœ— *Bun will achieve 20% developer adoption in our annual survey.* Wrong. Bun reached 14% in our December 2025 survey. + +5. โœ“ *At least three well-funded AI agent companies from 2023 will shut down or be acqui-hired without significant returns.* Correct. We counted six. + +6. โœ“ *Python will remain the most commonly used language in our developer survey.* Correct. + +7. ~ *The SBOM requirement in the EU Cyber Resilience Act will be delayed.* Partly correct โ€” there was a delayed phase-in, but the core requirement launched on schedule. + +8. โœ— *A major cloud provider will launch a WebAssembly-native function runtime to compete with containers.* Wrong. Progress was made but no major commercial launch. + +9. โœ“ *GitHub will lose market share in our developer survey's primary VCS platform question.* Correct โ€” from 83% to 79%. + +10. โœ“ *Developer burnout rates in our survey will remain above 35%.* Correct. They increased. + +Score: 6 correct, 2 wrong, 2 partial. About what we'd expect from honest, specific predictions. + +--- + +## 2026 Predictions + +**Prediction 1: WebAssembly will ship in production at a top-20 enterprise software company as a primary execution substrate.** + +Not as an experiment. Not as a components layer wrapping existing infrastructure. As the primary way a major production workload runs. The maturity of the tooling, the improvements in WASI P2, and the security benefits are reaching the threshold where a risk-averse enterprise can justify the migration. At least one will make the jump in 2026. + +**Prediction 2: The first major AI coding assistant data breach will occur and change how enterprises procure AI tools.** + +The amount of code that AI coding assistants have access to โ€” including proprietary algorithms, credentials accidentally included in context, and architectural information โ€” is an enormous and underappreciated attack surface. The security practices of AI tool providers are not uniformly rigorous. We predict a significant data incident involving AI coding tool infrastructure will occur in 2026. This will not kill the category but will significantly accelerate enterprise requirements for on-premise or single-tenant deployment options. + +**Prediction 3: Rust adoption in our developer survey will cross 25%.** + +Rust has grown from 13% in 2023 to 19% in 2024 to somewhere around 23% in 2025 (full data in our upcoming survey). The trajectory has been consistent and the reasons for it are structural: a generation of developers who learned Rust in university and through the wave of Rust evangelism in the mid-2020s is reaching positions of influence in engineering organisations. We expect the line to keep moving in 2026. + +**Prediction 4: At least one major open source project will adopt a maintainer certification system backed by a cryptographic identity.** + +The xz incident continues to shape thinking about maintainer vetting. The mechanism that was missing โ€” a way to establish that a new contributor is who they claim to be, with appropriate cryptographic binding โ€” has been discussed extensively but not shipped. We predict that at least one prominent open source project will deploy something in this space in 2026. + +**Prediction 5: The IPO of a developer tools company will be the most successful tech IPO of the year.** + +After the application-layer AI valuation compression, the most credible IPO candidates are companies with real, durable, growing revenue and genuine product-market fit. Developer tools companies meet these criteria better than most. The market has re-learned to value profitability and sustainable growth. A developer tools IPO will benefit from this re-learning. + +**Prediction 6: At least one major programming language will ship an official AI-assisted standard library function generator.** + +The pattern of "AI completes code I'm writing" is being extended to "AI generates idiomatic code for well-defined standard tasks." We expect a major language community โ€” likely Python, Go, or Kotlin โ€” to ship an official tool that generates standard library usage examples or fills common patterns using AI, integrated into the official toolchain. + +**Prediction 7: The term "vibe coding" will not appear in a serious engineering job description.** + +A backlash to the AI-generated code practices that became fashionable in 2024-2025 will produce employer differentiation โ€” companies that want engineers who understand their code will begin explicitly signalling this in hiring requirements. The phrase used informally to describe AI-dependent development will become a negative signal in professional contexts. + +**Prediction 8: The Linux Foundation will announce a funded maintenance programme for at least 50 critical open source packages.** + +The post-xz policy environment has created political will for sustained open source maintenance funding. The Sovereign Tech Fund model will be extended or replicated at larger scale. We expect the Linux Foundation, the most credible vehicle for industry-wide open source funding, to announce a programme funded by major technology companies that provides ongoing payment to maintainers of critical packages. + +**Prediction 9: A native macOS ARM port of a significant enterprise software product that has been Windows-or-Linux-only will ship.** + +The Apple Silicon platform has reached a level of developer market share that makes it untenable for enterprise tools to not support it. We predict at least one major enterprise software product that has historically been Windows-only or primarily Linux-focused will ship a native ARM macOS version. + +**Prediction 10: Our 2026 developer survey will show developer satisfaction with AI tools has declined compared to 2025, even as usage has increased.** + +The pattern of AI tool adoption producing subsequent dissatisfaction โ€” particularly among more experienced developers โ€” is a trend we have been tracking. Usage will increase because the tools are embedded in workflows and expensive to remove. Satisfaction will decline because the costs (reduced code understanding, quality concerns, dependency) have become clearer. Both things will be true simultaneously. + +--- + +Check back in December 2026 for the accounting. + +*Clara Winthorpe is open source and infrastructure editor at TechPulse.* diff --git a/sample-sites/techpulse/posts/2026-01-15-wasm-server.md b/sample-sites/techpulse/posts/2026-01-15-wasm-server.md new file mode 100644 index 0000000..4e030a4 --- /dev/null +++ b/sample-sites/techpulse/posts/2026-01-15-wasm-server.md @@ -0,0 +1,81 @@ +--- +title: "Server-Side WebAssembly Is Finally Ready for Production" +created: 2026-01-15 09:00 +author: Raj Patel +keywords: WebAssembly, WASM server, Spin framework, WASI P2, production, serverless, Fermyon +description: After years of "almost there," server-side WebAssembly has reached production readiness. We have the benchmark data, real company case studies, and an honest assessment of the remaining rough edges. +--- + +In late 2025, something shifted in the server-side WebAssembly ecosystem. Not a single dramatic event, but a convergence: the Spin framework reached v3.0, Fermyon Cloud achieved five-nines uptime for three consecutive quarters, the WASI P2 specification stabilised, and several companies whose names you would recognise began running WebAssembly workloads in production not as experiments but as infrastructure they would have to explain an outage for. + +We have spent the past several weeks benchmarking, building, and talking to the teams that are shipping production WebAssembly. Our conclusion: server-side WebAssembly is ready for production, with meaningful caveats about what "production" means in this context and what the rough edges are. + +## What Has Changed Since 2024 + +The WebAssembly server story has been "almost ready" for several years. What changed in 2025? + +**WASI P2 stabilised and shipped in major runtimes.** The WebAssembly System Interface Preview 2 โ€” the version of the interface standard that includes proper networking, the component model, and a more complete POSIX-like capability set โ€” shipped stable implementations in Wasmtime, Wasmer, and WasmEdge. Previous versions of WASI had significant limitations (most notably, no network sockets) that made them unsuitable for server applications. WASI P2 removes those limitations. + +**Spin v3 simplified the developer experience significantly.** Spin, the developer-friendly WebAssembly application framework from Fermyon, underwent a major revision that made it substantially easier to build production-ready applications. The local development experience โ€” running Spin applications locally with fast iteration, debugging support, and test tooling โ€” reached parity with frameworks like Express or FastAPI in terms of developer ergonomics. + +**Component composition tooling matured.** The `wasm-tools` suite and the `cargo component` toolchain for Rust (the primary language for production WebAssembly) reached stable, reliable releases. Building, composing, and deploying WebAssembly components no longer requires debugging cryptic toolchain errors. + +**Companies started shipping.** Several companies began running real user traffic through WebAssembly components. This is important not just as validation but as a signal: when companies with users depending on their software choose WebAssembly, the ecosystem has reached a level of reliability where that choice is defensible. + +## Benchmark Data + +We ran performance benchmarks comparing server-side WebAssembly (Spin on Wasmtime) against Node.js, Rust Axum, and Python FastAPI for a representative HTTP application: JSON API with a database read. + +**Cold start latency** (time to first response for a newly started instance): +- Spin/WebAssembly: 1.2ms +- Rust Axum: 8ms (process startup) +- Node.js: 145ms +- Python FastAPI: 340ms + +**Throughput** (requests/second, steady state, 100 concurrent connections): +- Rust Axum: 198,000 req/s +- Spin/WebAssembly: 87,000 req/s +- Node.js: 61,000 req/s +- Python FastAPI: 24,000 req/s + +**Memory per instance:** +- Spin/WebAssembly: 12MB baseline +- Node.js: 68MB baseline +- Python FastAPI: 85MB baseline +- Rust Axum: 6MB baseline + +The WebAssembly numbers are impressive, particularly cold start and memory. The throughput is lower than native Rust, which is expected โ€” there is a cost to the safety abstractions and the WebAssembly execution model โ€” but significantly higher than Node.js, which makes it competitive for latency-sensitive applications where the cold start and memory characteristics are advantages. + +## Who Is Running It in Production + +We spoke with teams at three companies running WebAssembly in production: + +**A European developer tools company** migrated their plugin execution sandbox from Node.js to WebAssembly components in Q4 2025. They run untrusted user-provided code (JavaScript and Python scripts) in their product; the security isolation of WebAssembly is the primary motivation. "The sandbox is our most important security boundary," their infrastructure lead told us. "WebAssembly's memory isolation and capability-based security model is significantly stronger than what we could achieve with a process-based sandbox, with a fraction of the overhead." + +**A media company** migrated their content personalisation service โ€” a stateless API that processes requests at the edge โ€” to Spin on Fermyon Cloud. They cited cold start latency and cost as the primary motivations. "We had a Node.js function that was taking 400ms cold start. That was acceptable but not great. The WebAssembly version starts in under 2ms. The cost difference at our scale is material." + +**A security software company** builds a vulnerability scanning tool that needs to run plugins in a sandboxed environment. They use WebAssembly components for plugin isolation. The primary attraction is the combination of security isolation and performance โ€” they can run hundreds of plugin instances concurrently on hardware that would struggle with equivalent process-based isolation. + +## The Remaining Rough Edges + +Production readiness does not mean perfect. The honest picture of server-side WebAssembly in early 2026 includes meaningful rough edges. + +**Language support is still Rust-centric.** Rust has the best WebAssembly toolchain, the best component model support, and the most documentation for production use. Go support has improved but has a larger binary size overhead. Python and JavaScript can run inside WebAssembly via interpreter embedding, but the resulting binaries are larger and the startup advantage narrows. If your team does not write Rust, the production WebAssembly story is less compelling. + +**Debugging is still harder than traditional environments.** WebAssembly debugging has improved significantly with DWARF extensions and improved Wasmtime debugging support, but stepping through a WebAssembly program in a debugger is still more effort than debugging native code. For teams that rely heavily on traditional debugging workflows, this is a meaningful adjustment. + +**The ecosystem of compatible libraries is smaller.** WebAssembly programs cannot use arbitrary native libraries โ€” they need either pure WebAssembly ports or WASM-compatible versions. For most web service use cases the available libraries are sufficient; for workloads that require specialised native code (graphics, certain cryptographic operations, hardware interfaces), you will hit gaps. + +**Persistent storage patterns are still evolving.** Stateless services on WebAssembly are mature. Patterns for working with persistent storage โ€” databases, object stores โ€” from WebAssembly are functional but the abstractions are still evolving, particularly for the component model's approach to resource handles. + +## The Assessment + +Server-side WebAssembly is production-ready for a specific class of applications: stateless or near-stateless services where cold start latency, memory density, and security isolation are important. For teams working in Rust, it is compelling today. For teams in other languages, "almost there" is becoming increasingly accurate, but "there" is not quite here yet. + +The convergence of events in 2025 has moved server-side WebAssembly from "promising experiment" to "credible production option." Teams building new serverless infrastructure or edge computing applications should evaluate it seriously. Teams with existing production systems have no urgent reason to migrate unless their current stack has problems that WebAssembly specifically solves. + +The technology is ready. The ecosystem is maturing. The adoption question now is as much about workflow change and team learning curves as it is about technical readiness. + +--- + +*Benchmark testing conducted January 2026 on dedicated hardware. Production case studies are from anonymous interviews with engineering teams. Raj Patel covers developer tools and infrastructure at TechPulse.* diff --git a/sample-sites/techpulse/posts/2026-02-28-open-source-llm-2026.md b/sample-sites/techpulse/posts/2026-02-28-open-source-llm-2026.md new file mode 100644 index 0000000..9bfef67 --- /dev/null +++ b/sample-sites/techpulse/posts/2026-02-28-open-source-llm-2026.md @@ -0,0 +1,105 @@ +--- +title: "The State of Open Source LLMs: A 2026 Benchmark" +created: 2026-02-28 14:00 +author: Raj Patel +keywords: open source LLMs, LLM benchmarks, Llama, Mistral, AI models 2026, inference, enterprise AI +description: We benchmarked 12 open-weight language models across reasoning, generation, cost, and deployment complexity. Here is the honest 2026 state of play. +--- + +Every few months the open source LLM landscape shifts dramatically enough to warrant reassessment. A model that was state-of-the-art in October may be mediocre by March. The benchmarking work is genuinely useful because the improvement trajectory is steep and the specific rankings matter for real deployment decisions. + +We spent six weeks running a comprehensive benchmark suite against 12 open-weight models, with a focus on the criteria that matter for production deployment: reasoning capability, text generation quality, cost per token at production scale, deployment complexity, and enterprise readiness characteristics (compliance, data governance, predictable behaviour). + +## The Models Tested + +We tested: Llama 3.3 70B, Llama 3.3 8B, Mistral Large 2, Mistral 7B, Gemma 3 27B, Gemma 3 7B, DeepSeek R2 Distill, Qwen 2.5 72B, Phi-4, Falcon 3 40B, OLMo 2 7B, and Command R+. + +All tests used quantized models where applicable to model realistic deployment scenarios. Inference was run on A100 80GB GPUs; cost calculations are based on spot instance pricing on AWS and GCP as of February 2026. + +## Reasoning Performance + +For reasoning tasks โ€” mathematical problem solving, logical deduction, multi-step code debugging, structured analysis โ€” the rankings are more differentiated than for simple generation tasks: + +**Tier 1 (competitive with GPT-4 class reasoning on many tasks):** +- Llama 3.3 70B: Exceptionally strong reasoning for its class. On MATH-500 (competition-level math), scored 73.2%. +- DeepSeek R2 Distill: The standout in this benchmark cycle. Achieves Llama 3.3 70B-level reasoning at 7B parameters by distilling reasoning traces from a larger teacher model. MATH-500: 71.8%. +- Qwen 2.5 72B: Strong mathematical reasoning, particularly notable for Chinese language tasks. MATH-500: 74.1%. + +**Tier 2 (solid reasoning, notable tradeoffs):** +- Mistral Large 2: Reliable but not exceptional on pure reasoning. Better on instruction following than logic problems. +- Gemma 3 27B: Better than its size class on reasoning, competitive with Llama 70B on many tasks at lower computational cost. + +**Tier 3 (limited to simpler reasoning tasks):** +- All 7B class models except DeepSeek R2 Distill: adequate for simple multi-step problems, unreliable on competition-level math or complex code debugging. + +## Text Generation Quality + +For text generation โ€” writing, summarisation, translation, following complex instructions โ€” the quality gap between models is smaller than the reasoning gap: + +Llama 3.3 70B, Mistral Large 2, and Qwen 2.5 72B are all competitive with each other on standard generation tasks. The differentiation comes from style: Llama 3.3 tends toward somewhat more formal outputs; Mistral Large 2 is notably strong on structured outputs (JSON, formatted reports); Qwen 2.5 72B has superior multilingual performance. + +The 7B class models (Mistral 7B, Gemma 3 7B, Llama 3.3 8B) perform surprisingly well on simple generation tasks โ€” blog posts, email drafts, basic summarisation โ€” where the gap with larger models is less apparent to non-expert evaluators. + +## Cost Per Token at Production Scale + +Cost per 1 million tokens (combined input + output, assuming 70% input / 30% output ratio, at AWS spot prices): + +- Llama 3.3 8B (quantized): $0.08 +- Mistral 7B (quantized): $0.09 +- DeepSeek R2 Distill (quantized): $0.11 +- Gemma 3 7B: $0.10 +- Phi-4 (14B): $0.18 +- Gemma 3 27B: $0.29 +- Llama 3.3 70B (quantized): $0.51 +- Mistral Large 2: $0.67 +- Qwen 2.5 72B: $0.55 +- Falcon 3 40B: $0.38 + +For context, GPT-4o API pricing from OpenAI is approximately $5-10 per million tokens depending on the tier, and Claude's API is similar. The cost advantage of self-hosted open models ranges from 7x to 70x depending on the model choice and the commercial API being compared. + +This cost differential is the primary driver of enterprise adoption of open-weight models. For high-volume inference โ€” customer support, document processing, code completion at scale โ€” the economics of self-hosting are compelling even accounting for infrastructure and operational costs. + +## Deployment Complexity + +Not all models are equally easy to deploy. We assessed each model on the effort required to go from "we want to run this in production" to "it is running in production": + +**Straightforward deployment (3 days or less for a team with basic MLOps knowledge):** +- Llama 3.3 8B: Excellent documentation, multiple inference server options (vLLM, Ollama, llama.cpp), quantized versions widely available. +- Mistral 7B: Similar to Llama; extensive community support. +- Phi-4: Microsoft's documentation and tooling are good; easy to deploy via Azure ML or self-hosted. + +**Moderate deployment complexity (1-2 weeks):** +- Llama 3.3 70B: Requires multi-GPU setup (2x A100 minimum for quantized). Documentation is good but hardware requirements add complexity. +- Gemma 3 27B: Google's tooling is less mature than Meta's for self-hosting; requires more custom configuration. +- Mistral Large 2: Large model (123B parameters); requires significant hardware and careful quantization setup. + +**More complex (potentially weeks with team knowledge gaps):** +- DeepSeek R2 Distill: Chinese documentation is primary; English translation available but sometimes lags. +- Falcon 3 40B: Less community support than Meta and Google models; fewer ready-made tooling options. +- OLMo 2: Most "truly open" model (training data included), but less polished deployment experience. + +## Enterprise Readiness Assessment + +For enterprises evaluating open-weight models, several characteristics beyond benchmark scores matter: + +**Predictability of outputs** โ€” models that reliably follow structured output instructions, maintain consistent persona, and do not produce unexpected outputs under edge cases. Llama 3.3 and Mistral models score well here. + +**Safety and compliance** โ€” whether the model has meaningful safety tuning and whether it respects access restrictions (system prompts preventing specific outputs). This is an area where all open models lag proprietary models, which have had more investment in RLHF and safety fine-tuning. + +**Data residency and governance** โ€” the primary reason enterprises adopt open-weight models. Self-hosted models offer complete data governance; API-based models do not. + +**Fine-tuning support** โ€” the ability to fine-tune the model on proprietary data to improve domain performance. All tested models support LoRA/QLoRA fine-tuning. + +## The Bottom Line + +The open-weight LLM landscape in February 2026 is substantially better than it was a year ago. The quality gap between open models and frontier proprietary models has narrowed for most common tasks. For enterprises with specific use cases โ€” document processing, domain-specific generation, code assistance โ€” fine-tuned open models often meet or exceed the performance of general-purpose proprietary models at a fraction of the cost. + +For the hardest reasoning tasks โ€” complex mathematical problem solving, multi-step logical deduction, research synthesis โ€” frontier proprietary models still lead. The gap is narrowing but has not closed. + +The practical recommendation depends on use case. For high-volume, domain-specific tasks where cost matters and data governance is required: deploy Llama 3.3 70B or Qwen 2.5 72B with appropriate hardware, invest in fine-tuning. For reasoning-intensive tasks where quality is paramount: the commercial frontier models remain the better choice, with the cost premium justified by capability. + +The bifurcated market is here to stay. + +--- + +*Benchmarks conducted February 2026. Full benchmark methodology and raw data available to TechPulse subscribers.* diff --git a/sample-sites/techpulse/posts/2026-03-20-platform-consolidation.md b/sample-sites/techpulse/posts/2026-03-20-platform-consolidation.md new file mode 100644 index 0000000..653099e --- /dev/null +++ b/sample-sites/techpulse/posts/2026-03-20-platform-consolidation.md @@ -0,0 +1,69 @@ +--- +title: "Developer Platform Consolidation: The End of the Microtools Era?" +created: 2026-03-20 10:30 +author: Maya Osei +keywords: developer platforms, GitHub, GitLab, Vercel, AWS, platform consolidation, developer tools market +description: Our developer survey data shows accelerating consolidation in developer tooling. We examine who is winning, who is losing, and what developers actually want from the platforms they use. +--- + +The developer tools market of 2018-2022 was defined by proliferation. The "best-of-breed" philosophy argued that developers should assemble their own tool stacks from specialised components, each class of problem addressed by a dedicated product with deep expertise in that specific area. Version control was GitHub. CI/CD was CircleCI or GitHub Actions. Deployment was Netlify or Vercel. Observability was Datadog. Alerting was PagerDuty. Error tracking was Sentry. Security scanning was Snyk. Each category had a leader, a challenger, and several hopeful entrants. + +The data from our 2026 developer survey suggests that this era is ending โ€” or at least maturing into something different. Consolidation is accelerating. Developers are increasingly using fewer platforms that do more things, and the buying patterns of engineering organisations are shifting toward platforms over point solutions. + +## The GitHub/GitLab/Gitea Market Structure + +In our 2026 survey, 79% of respondents use GitHub as their primary VCS platform, down from 83% in 2025 and 86% in 2023. The decline is gradual but consistent. GitLab's share has held roughly flat at 14-15%. Gitea/Forgejo (the open source self-hosted option) has grown from 2% to 5% over the same period, driven primarily by organisations with compliance requirements or political preferences for self-hosted infrastructure. + +The GitHub decline deserves nuance. It has not lost users in absolute terms โ€” the overall developer population is growing, and GitHub continues to grow. What the survey data suggests is that the category that was once essentially synonymous with GitHub now has genuine competition that a meaningful minority of developers prefers. + +The primary driver of GitHub share loss is bundling competition from GitLab and, to a lesser extent, Bitbucket and Gitea. GitLab offers a deeply integrated platform that includes CI/CD, security scanning, container registry, and deployment pipelines without requiring external integrations. For teams that prefer one vendor to many, GitLab's value proposition has strengthened as GitHub's own integrated offerings (Actions, Advanced Security, Copilot) have grown more expensive and complex. + +"We evaluated GitHub versus GitLab last year," one engineering director told us. "GitHub has better individual components โ€” the developer experience on GitHub.com is better. GitLab has better total integration. For our compliance requirements, the single platform story won." + +## Vercel vs. AWS: The Frontend-to-Fullstack Spectrum + +Vercel's growth story over the past five years has been one of the most impressive in developer tools. They built a deployment platform for frontend developers that was substantially simpler to use than AWS, priced accessibly, and built out an ecosystem around Next.js (which they also develop) that created significant switching costs. + +In our 2026 survey, 28% of respondents who deploy web applications use Vercel for at least some production workloads. That is a real number. But Vercel's challenge โ€” and it is one they have been working on publicly โ€” is that their pricing scales badly with production usage. The companies we spoke with that have had the most mixed experiences with Vercel describe the same pattern: excellent for small and medium scale, expensive at large scale, and constrained in the degree of infrastructure control available. + +AWS's position is different: it is the default for enterprises and for workloads that require flexibility or scale that managed platforms cannot provide. AWS's developer experience has improved substantially with the growth of services like App Runner, Lambda, and the managed database services, but it remains complex compared to Vercel or Fly.io for straightforward web application deployment. + +The market has not converged on a single answer because the answer is use-case dependent. Vercel is optimal for frontend-heavy applications with predictable traffic patterns. AWS is optimal for complex backend infrastructure with unusual requirements. The developers who are most frustrated are those whose requirements sit in the middle โ€” fullstack applications with backend complexity โ€” and who have to either accept Vercel's constraints or pay AWS's complexity tax. + +## What Developers Actually Want + +We asked our survey respondents to rank the most important characteristics of their primary development platform. The results: + +1. **Reliability** (selected by 81% as among their top three): The number-one concern is whether the platform is up when they need it. Developers have been burned by platform outages and have long memories. + +2. **Pricing predictability** (73%): The second most important concern is whether they can predict what they will pay. Services with usage-based pricing that scales unpredictably are increasingly disliked; several respondents mentioned specific incidents where a traffic spike produced a bill they did not expect. + +3. **DX quality** (69%): Developer experience โ€” documentation quality, CLI ergonomics, local development experience โ€” is the third most important factor. This is where newer platforms like Vercel, Fly.io, and Railway have competed successfully with AWS and GCP. + +4. **Integration breadth** (58%): The ability to integrate with the other tools in the developer's stack. + +5. **Vendor independence** (44%): A meaningful minority prioritise platforms that they are not locked into โ€” open source platforms, standards-compliant APIs, or platforms with strong export capabilities. + +The gap between reliability/predictability (top two) and DX quality (third) is significant. Developers want platforms that work reliably and predictably first; they want them to be pleasant to use second. The implication for newer platforms is that their DX advantage, while real, is not sufficient to win against incumbents that are significantly more reliable. + +## Winners and Losers in the Consolidation + +The consolidation dynamic creates winners and losers that do not always map to the quality of the product. + +**Winners:** GitHub (despite losing market share, it is consolidating CI/CD, security, and AI tooling on a single platform); AWS (as complexity increases, enterprises default to what they know); GitLab (the integrated platform story is genuinely stronger than it was); Datadog (observability consolidation is ongoing). + +**Losers:** Independent CI/CD services (CircleCI, Buildkite, Jenkins are losing to GitHub Actions and GitLab CI for new workloads); individual open source security scanners (being replaced by bundled GitHub Advanced Security or GitLab security features); niche project management tools (being absorbed by Linear and similar tools that have expanded scope). + +**Uncertain:** Vercel (the platform is genuinely excellent but the pricing model is a persistent concern that limits growth in the enterprise segment); Fly.io (strong developer love but the reliability history creates enterprise hesitation); Railway (growing fast in small company segment but limited data on enterprise trajectory). + +## What It Means for Developers + +The consolidation trend has real implications for individual developers. Fewer platform choices means less ability to mix-and-match the best tool for each job, but also less cognitive load from maintaining many integrations. The best-of-breed era's promise โ€” that you could assemble an optimal tool stack โ€” was real, but so was its cost: every integration is a surface area for failure, every API key is an operational concern. + +The developers who benefit most from consolidation are those who want to minimise time spent on tooling and maximise time spent on product work. The developers who lose are those with highly specific needs that the consolidated platforms do not address well. + +Neither outcome is obviously better. It is, as ever, a tradeoff. + +--- + +*Maya Osei covers developer platform economics at TechPulse. Survey data from the TechPulse 2026 Developer Survey (n=4,200). Company interviews conducted February-March 2026.* diff --git a/sample-sites/techpulse/posts/2026-05-01-developer-survey-2026.md b/sample-sites/techpulse/posts/2026-05-01-developer-survey-2026.md new file mode 100644 index 0000000..5c1465f --- /dev/null +++ b/sample-sites/techpulse/posts/2026-05-01-developer-survey-2026.md @@ -0,0 +1,106 @@ +--- +title: "TechPulse Developer Survey 2026: AI Has Won, But Developers Have Mixed Feelings" +created: 2026-05-01 09:00 +author: Maya Osei +keywords: developer survey 2026, AI tools, deskilling, developer productivity, programming languages, AI adoption +description: Our 2026 survey of 4,200 developers shows 78% use AI tools daily. The adoption is near-universal among younger developers โ€” but the concerns about deskilling, quality, and dependency are louder than ever. +--- + +The 2026 TechPulse Developer Survey โ€” our fifth annual โ€” is our largest yet: 4,200 respondents from 73 countries, with balanced representation across company sizes, experience levels, and industries. We ran the survey for four weeks in March and April 2026. + +The headline is stark: AI coding tools have reached near-ubiquitous adoption. 78% of respondents use AI coding tools daily, up from 73% in 2024. Among developers with five or fewer years of experience, the number is 91%. Among developers with fifteen or more years of experience, it is 61%. + +The experience gap is telling. Younger developers have grown up with these tools and find it hard to imagine working without them. Experienced developers are more likely to use them selectively and more likely to express ambivalence about what they are doing to the field. + +## The Adoption Numbers + +**Daily AI tool usage by experience:** +- 0-2 years: 94% +- 3-5 years: 89% +- 6-10 years: 79% +- 11-15 years: 67% +- 16+ years: 61% + +**Primary AI tool used:** +- GitHub Copilot (individual or enterprise): 38% +- Cursor: 29% +- JetBrains AI Assistant: 14% +- Claude in IDE/API: 8% +- Custom/self-hosted LLM integration: 11% +- Other: 7% +(Multiple selection allowed) + +Cursor's rise is the most significant tool market shift in the past two years. It now rivals GitHub Copilot for daily use, having grown from 22% in 2024. The growth is driven by the AI-native editor's deeper integration: Cursor provides contextual awareness of entire codebases, not just the current file, and its "ask" features allow natural language interaction with the codebase at a level that Copilot's autocomplete model does not match. + +## Satisfaction vs. Usage: The Widening Gap + +The most striking finding in the 2026 survey is the divergence between usage rates and satisfaction rates โ€” a gap we first noticed emerging in 2025 data and that has widened significantly. + +**Usage vs. satisfaction for AI tools:** +- Daily users: 78% +- "Very satisfied" with AI tools: 31% +- "Somewhat satisfied": 44% +- "Mixed feelings": 18% +- "Primarily dissatisfied": 7% + +"Mixed feelings" increased from 13% in 2025 to 18% in 2026. "Very satisfied" decreased from 38% to 31%. The primary satisfaction driver is productivity on certain tasks (very high satisfaction); the primary dissatisfaction driver is quality concerns and the accumulating sense of not understanding one's own code (increasing strongly). + +## Concerns About Deskilling + +For the first time, we included specific questions about deskilling โ€” the concern that relying on AI tools may be degrading underlying programming skills. The responses were striking. + +**"Using AI coding tools has reduced my ability to write code without them":** +- Strongly agree: 14% +- Somewhat agree: 31% +- Neither agree nor disagree: 22% +- Somewhat disagree: 24% +- Strongly disagree: 9% + +45% of respondents agree to some degree that AI tools have reduced their ability to work without them. This is a form of dependency that most productivity tools do not produce โ€” using a calculator does not impair arithmetic ability, but many respondents believe AI coding assistance has impaired their coding ability. + +The agreement rate is highest among developers with 3-7 years of experience โ€” the group that adopted AI tools early in their career and has now had them long enough to notice an effect. It is lowest among senior developers who adopted them selectively and latest. + +Several open response comments crystallise the concern: + +*"I started my career writing everything by hand. I can still do it if I must. My colleagues who started two years ago struggle to write a for loop without autocomplete. That is a problem I am not sure the industry is taking seriously."* โ€” Software engineer, 14 years experience + +*"I caught myself googling the syntax for something I should know by heart. I used to know it by heart. Copilot has been doing it for me for 18 months and I've forgotten."* โ€” Backend developer, 6 years experience + +*"I don't think I'm getting deskilled. I think I'm getting reskilled โ€” the skills that matter are changing. Understanding code, architecture, and debugging still require the same skills as before. AI does the mechanical writing part. That seems fine to me."* โ€” Principal engineer, 11 years experience + +## Salary Impacts + +We asked a new question in 2026: whether respondents believe AI tools have affected their salary leverage. The results are complicated. + +**Effect of AI tools on salary leverage:** +- Increased leverage (AI makes me more valuable): 29% +- No effect on salary leverage: 43% +- Decreased leverage (AI reduces the premium on individual skills): 28% + +A near-even split between "makes me more valuable" and "makes me less valuable." The interpretation of respondents who feel AI increases their leverage: AI multiplies the output of skilled developers, making experienced developers who can use AI effectively more valuable. The interpretation of respondents who feel AI decreases their leverage: AI reduces the value of junior programming work, compresses salaries for entry-level positions, and is beginning to reduce the perceived value of individual technical skill. + +Both effects are probably real for different segments of the market. Senior developers who use AI effectively to multiply their output are commanding premiums. Entry-level programming positions are reported by respondents to be harder to find and less well-compensated than two years ago. + +## What Developers Wish AI Could Do Better + +We asked respondents what they wish AI coding tools did better. The top five responses (ranked by frequency): + +1. **Better understanding of the existing codebase context** (cited by 64%): The frustration that AI suggestions do not account for project-specific patterns, architecture decisions, and constraints. + +2. **More honest about uncertainty** (51%): The confident-but-wrong failure mode. Respondents want AI tools that say "I'm not sure" rather than generating plausible-sounding incorrect code. + +3. **Better debugging assistance** (48%): The gap between AI's ability to generate code and its ability to diagnose problems in existing code remains frustrating. + +4. **Security awareness** (38%): Respondents want AI tools that flag security concerns while generating code, rather than producing code that passes tests but has security issues. + +5. **Better handling of legacy code** (35%): AI tools trained primarily on modern, idiomatic code struggle with legacy codebases, which is where many professional developers spend most of their time. + +## The Burnout Picture + +Burnout rates remain elevated: 39% of respondents describe significant burnout in the past 12 months, a slight increase from 38% in 2024. The causes have shifted: concern about AI's impact on job security has entered the top five causes of developer stress for the first time, alongside the previously dominant factors of meeting load, understaffing, on-call stress, and technical debt pressure. + +The developers least likely to report burnout are those who feel they have agency over their AI tool usage โ€” who use them selectively, understand their outputs, and maintain skills that do not depend on them. The developers most likely to report burnout related to AI are those who feel pressure to adopt tools they have concerns about, and those who feel the pace of AI-driven change in the field is making their existing skills obsolete faster than they can adapt. + +--- + +*Full methodology, raw data, and cross-tabulations available to TechPulse subscribers. Survey conducted March 17 โ€“ April 14, 2026. n=4,200 qualified respondents.* diff --git a/sample-sites/techpulse/search.json b/sample-sites/techpulse/search.json new file mode 100644 index 0000000..ba01dff --- /dev/null +++ b/sample-sites/techpulse/search.json @@ -0,0 +1,302 @@ +[ + { + "file": "pages/home.md", + "title": "Home", + "section-id": "site", + "keywords": "technology news, developer tools, AI, open source, startups, cloud", + "description": "TechPulse โ€” independent technology news and analysis for developers and tech professionals", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "Welcome to TechPulse. TechPulse is an independent technology news and analysis publication built for the people who actually build things. We cover artificial intelligence, open source software, developer tools, cloud infrastructure, startups, and the business of technology โ€” without the hype cycles, vendor press releases, or breathless trend-chasing that defines too much of tech media." + }, + { + "file": "pages/about.md", + "title": "About TechPulse", + "section-id": "site", + "keywords": "about TechPulse, editorial team, mission, advertising policy, independent journalism", + "description": "The story of TechPulse, our editorial mission, team bios, and advertising policy", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "TechPulse is an independent technology news and analysis publication. We launched in May 2021 with the conviction that tech media had a problem. Our editorial team: Maya Osei (editor), Raj Patel (AI and developer tools correspondent), Clara Winthorpe (open source editor). We are editorially independent. We do not produce sponsored content." + }, + { + "file": "pages/newsletter.md", + "title": "Newsletter", + "section-id": "site", + "keywords": "newsletter, subscribe, weekly digest, email", + "description": "Subscribe to the TechPulse weekly newsletter โ€” technology news and analysis in your inbox", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "Every Saturday morning, the TechPulse newsletter lands in subscriber inboxes with a carefully curated week in technology. No clickbait. No press release reprints. Just the stories that mattered, explained with context. Free tier: Weekly digest. Subscriber tier: Monthly deep-dives and full archive access." + }, + { + "file": "pages/topics.md", + "title": "Topics", + "section-id": "site", + "keywords": "AI, open source, developer tools, startups, cloud, security, hardware, technology coverage", + "description": "An overview of TechPulse's coverage areas โ€” AI/ML, open source, developer tools, startups, cloud, security, and hardware", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "TechPulse maintains sustained coverage across seven core areas of technology: Artificial Intelligence and Machine Learning, Open Source, Developer Tools, Startups and Funding, Cloud and Infrastructure, Security, and Hardware. We focus on beats where we have expertise and source networks to report with genuine depth." + }, + { + "file": "pages/archive.md", + "title": "Archive", + "section-id": "site", + "keywords": "archive, all articles, technology news archive", + "description": "The complete TechPulse article archive โ€” all posts in reverse chronological order", + "author": null, + "date": "", + "datetime": "", + "language": "en", + "body": "The complete TechPulse archive, updated with every new publication. Our coverage spans AI and machine learning, open source, developer tools, startups and funding, cloud infrastructure, and security." + }, + { + "file": "posts/2024-03-12-llm-coding-assistants.md", + "title": "The Real Impact of AI Coding Assistants on Developer Productivity", + "section-id": null, + "keywords": "AI coding assistants, GitHub Copilot, developer productivity, code quality, security vulnerabilities", + "description": "A study of 500 developers reveals a 40% productivity gain from AI coding tools โ€” but the picture is more complicated than that number suggests.", + "author": "Raj Patel", + "date": "2024-03-12", + "datetime": "2024-03-12 09:00", + "language": "en", + "body": "Study of 500 developers across 40 companies shows 40% productivity gain from AI coding assistants. But code quality concerns emerge: review times increased 23%, rework rates increased from 18% to 29%. Security vulnerabilities in AI-generated code found at 1.8x baseline rate. Concerns about developer dependency and skill development, especially for junior engineers." + }, + { + "file": "posts/2024-05-20-open-source-sustainability.md", + "title": "Open Source Sustainability Crisis: Who Pays for the Infrastructure?", + "section-id": null, + "keywords": "open source, sustainability, xz backdoor, OpenSSF, Sovereign Tech Fund, funding", + "description": "The xz backdoor incident exposed what many already knew โ€” the open source infrastructure powering global commerce is maintained by a handful of burned-out volunteers.", + "author": "Clara Winthorpe", + "date": "2024-05-20", + "datetime": "2024-05-20 14:00", + "language": "en", + "body": "The xz backdoor incident revealed a burned-out solo maintainer targeted by a sophisticated social engineering attack. OpenSSF, Sovereign Tech Fund, and GitHub Sponsors provide partial solutions. Three structural models: infrastructure levy, procurement mandate, and foundation consolidation. The real risk is not neglect but targeted attacks against exhausted maintainers." + }, + { + "file": "posts/2024-07-08-rust-linux-kernel.md", + "title": "Rust in the Linux Kernel: One Year Later", + "section-id": null, + "keywords": "Rust, Linux kernel, kernel drivers, systems programming, memory safety, Linus Torvalds", + "description": "One year after the first Rust code landed in the Linux kernel, we assess what has merged, how developers have received it, and what the safety improvements look like.", + "author": "Clara Winthorpe", + "date": "2024-07-08", + "datetime": "2024-07-08 10:30", + "language": "en", + "body": "Approximately 31,000 lines of Rust in Linux kernel tree as of 6.9. Nova GPU driver merged. Developer reception has warmed. 65-70% of kernel CVEs historically are memory safety bugs that Rust prevents. Concerns about maintainer pool depth and Rust toolchain stability for long support periods remain." + }, + { + "file": "posts/2024-08-15-startup-ai-funding.md", + "title": "AI Startup Funding Hits $47B in H1 2024 โ€” But Where's the Revenue?", + "section-id": null, + "keywords": "AI funding, startup investment, venture capital, AI revenue, AI startups 2024", + "description": "AI startups raised $47 billion in the first half of 2024. Which categories received it, which companies are generating real revenue, and which are burning cash.", + "author": "Maya Osei", + "date": "2024-08-15", + "datetime": "2024-08-15 11:00", + "language": "en", + "body": "$47 billion in AI startup funding H1 2024. Foundation model companies $14.2B. AI infrastructure $9.3B. Vertical AI applications $12.1B. Developer tools $7.4B. Agents $4B. Revenue quality concerns: ARR inflation, high churn, pilot customers counted as revenue. Investors applying 60-70% haircut to reported ARR internally." + }, + { + "file": "posts/2024-10-03-sqlite-everywhere.md", + "title": "The SQLite Revolution: How a 25-Year-Old Database Took Over the Cloud", + "section-id": null, + "keywords": "SQLite, Cloudflare D1, Turso, libSQL, edge computing, databases, cloud", + "description": "SQLite was designed for embedded systems. Somehow it has become the database of choice for the edge computing era.", + "author": "Raj Patel", + "date": "2024-10-03", + "datetime": "2024-10-03 09:00", + "language": "en", + "body": "SQLite powers Cloudflare D1, Turso, Bun's built-in database. Edge computing loves SQLite for in-process execution, no network overhead, single portable file. Cloudflare D1 replicates SQLite to hundreds of edge locations. libSQL fork adds replication and extensions. Limitation: single-writer concurrency model." + }, + { + "file": "posts/2024-11-18-wasm-components.md", + "title": "WebAssembly Components: The Runtime-Agnostic Future of Software", + "section-id": null, + "keywords": "WebAssembly, WASM, WASI, component model, ByteCode Alliance, containers, runtime", + "description": "WASI 0.2 and the WebAssembly component model represent a genuinely new approach to software packaging.", + "author": "Raj Patel", + "date": "2024-11-18", + "datetime": "2024-11-18 13:00", + "language": "en", + "body": "WASI 0.2 ships the component model enabling language-agnostic composition. WIT interface definition language for typed boundaries. Fastly, Fermyon Spin, Microsoft Azure, Shopify using in production. Microsecond startup vs. container milliseconds. Debugging tooling and language support gaps remain." + }, + { + "file": "posts/2024-12-05-developer-survey-2024.md", + "title": "TechPulse Developer Survey 2024: 3,000 Respondents, Key Findings", + "section-id": null, + "keywords": "developer survey 2024, programming languages, AI tools, remote work, salary, developer burnout", + "description": "Results from our annual survey of 3,000 developers โ€” language popularity, AI adoption, salary data, remote work trends, and burnout rates.", + "author": "Maya Osei", + "date": "2024-12-05", + "datetime": "2024-12-05 10:00", + "language": "en", + "body": "3,047 respondents, 61 countries. Python 67%, JavaScript 64%, Rust 19%, Go 31%. AI tool adoption 73% weekly. Cursor grew to 22% share. 51% fully remote. Median US salary 6-10 years: $153,000. Burnout: 38% significant burnout past 12 months. 18% trying to reduce AI tool usage." + }, + { + "file": "posts/2025-01-22-anthropic-o3.md", + "title": "Chain-of-Thought Models Change Everything โ€” But Not in the Way You Think", + "section-id": null, + "keywords": "reasoning models, chain-of-thought, o1, enterprise AI, LLM limitations, AI reasoning", + "description": "The new generation of reasoning models that think before answering have changed what AI can do โ€” but the change is more specific than the coverage suggests.", + "author": "Raj Patel", + "date": "2025-01-22", + "datetime": "2025-01-22 09:30", + "language": "en", + "body": "Chain-of-thought reasoning models improve on sustained multi-step reasoning tasks. Enterprise success in code review, legal document analysis, complex data analysis. Failure modes: confident wrong reasoning, long-horizon task limitations, domain knowledge gaps, 5-10x cost increase. Best use: decision tasks, not generation tasks." + }, + { + "file": "posts/2025-03-10-platform-engineering.md", + "title": "Platform Engineering Is the New DevOps โ€” And That's Both Good and Bad", + "section-id": null, + "keywords": "platform engineering, DevOps, internal developer platforms, CNCF, golden paths, developer experience", + "description": "Platform engineering has become the dominant framework for thinking about internal developer infrastructure. Is it solving the right problems?", + "author": "Maya Osei", + "date": "2025-03-10", + "datetime": "2025-03-10 14:00", + "language": "en", + "body": "CNCF 2025: 71% of 500+ engineer orgs have internal developer platforms. 34% faster deployment, 28% faster onboarding in mature orgs. Problems: platform teams as bottlenecks, golden paths becoming golden cages, complexity concentration. Best implementations measure DevEx systematically and build escape hatches for edge cases." + }, + { + "file": "posts/2025-04-28-open-source-ai-models.md", + "title": "Open Source AI Models in 2025: The Landscape Is More Complex Than It Seems", + "section-id": null, + "keywords": "open source AI, Llama 3, Mistral, Gemma, open weights, AI licensing, Meta AI", + "description": "Llama, Mistral, Gemma โ€” the 'open source AI' movement is growing fast. But what does 'open' actually mean when applied to large language models?", + "author": "Raj Patel", + "date": "2025-04-28", + "datetime": "2025-04-28 11:00", + "language": "en", + "body": "Most 'open source' AI models release only weights, not training data or code. Llama's custom licence prohibits training other LLMs. Mistral uses Apache 2.0. Gemma has custom use restrictions. BLOOM is closest to genuinely open. OSI's Open Source AI definition requires training data documentation, training code, and OSI-approved licence. Almost no current models qualify." + }, + { + "file": "posts/2025-06-15-kubernetes-fatigue.md", + "title": "Kubernetes Fatigue Is Real โ€” Here's What Teams Are Doing Instead", + "section-id": null, + "keywords": "Kubernetes, k8s, platform alternatives, Fly.io, Railway, managed services, infrastructure fatigue", + "description": "Survey of 200 engineering teams finds growing Kubernetes fatigue and a market in transition.", + "author": "Clara Winthorpe", + "date": "2025-06-15", + "datetime": "2025-06-15 09:00", + "language": "en", + "body": "200 teams surveyed: 18% say k8s more trouble than worth, 7% actively migrating away. Teams leaving are typically under 30 engineers with fewer than 10 services. Managed services (Fargate, Cloud Run), Fly.io, Railway gaining share. Satisfied k8s users have dedicated platform teams, genuine need for k8s capabilities, and abstraction layers above raw k8s." + }, + { + "file": "posts/2025-07-22-vc-funding-2025.md", + "title": "Tech Funding in 2025: The AI Bubble vs. The Infrastructure Boom", + "section-id": null, + "keywords": "venture capital 2025, AI funding, infrastructure investment, IPO market, tech funding trends", + "description": "H1 2025 funding data shows a bifurcated market โ€” AI application layer funding has cooled while infrastructure investment continues to grow.", + "author": "Maya Osei", + "date": "2025-07-22", + "datetime": "2025-07-22 10:30", + "language": "en", + "body": "$89B global VC in H1 2025, 14% below H1 2024. Application layer correction: ARR quality concerns, high churn. Infrastructure investment continues. IPO market thawing for profitable companies. LPs reducing early-stage generalist commitments. Consumer fintech, pure SaaS without AI, web3, no-code all declining." + }, + { + "file": "posts/2025-09-08-deno-v3.md", + "title": "Deno v3: Is Node.js Compatibility Finally Good Enough?", + "section-id": null, + "keywords": "Deno v3, Node.js, npm compatibility, JavaScript runtime, Bun, benchmarks", + "description": "Deno v3 launched with the strongest Node.js compatibility claim the project has ever made. We ran the benchmarks and talked to teams who have actually migrated.", + "author": "Clara Winthorpe", + "date": "2025-09-08", + "datetime": "2025-09-08 13:00", + "language": "en", + "body": "Deno v3 http throughput 97K req/s vs Node 89K vs Bun 142K. Startup 31ms vs Node 48ms vs Bun 12ms. 43 of 50 tested npm packages work. Native C++ modules remain problematic. Real migration stories: media company Express migration in 3 days, fintech declined due to native module dependency." + }, + { + "file": "posts/2025-10-14-security-supply-chain.md", + "title": "Software Supply Chain Security in 2025: Progress Report", + "section-id": null, + "keywords": "software supply chain security, SBOM, sigstore, GitHub security, dependencies, enterprise security", + "description": "Three years after the xz incident, we assess how much enterprise software supply chain security has actually improved.", + "author": "Raj Patel", + "date": "2025-10-14", + "datetime": "2025-10-14 09:00", + "language": "en", + "body": "SBOM adoption routine in regulated sectors but operationalisation remains weak. Sigstore now signs all PyPI packages, npm beginning adoption. Dependabot catches known CVEs but misses supply chain attacks. 12 enterprise interviews: automated scanning routine, SBOM ingestion still rare, build reproducibility early stage. Remaining gap: sophisticated targeted attacks against maintainers." + }, + { + "file": "posts/2025-11-30-ai-agents-enterprise.md", + "title": "AI Agents in the Enterprise: What Actually Works", + "section-id": null, + "keywords": "AI agents, enterprise AI, automation, LLM agents, autonomous AI, AI guardrails", + "description": "Case studies from five companies reveal what AI agents are reliably delivering in enterprise settings โ€” and why autonomous decision-making remains out of reach.", + "author": "Maya Osei", + "date": "2025-11-30", + "datetime": "2025-11-30 11:00", + "language": "en", + "body": "Five enterprise case studies: financial services document processing (60% faster), code review (30% issues caught), HR candidate screening (legal concerns), customer support (62% autonomous resolution), legal contract review. Common finding: all successful deployments are bounded with mandatory human review. Autonomous decision-making is where all five drew the line." + }, + { + "file": "posts/2025-12-20-developer-predictions-2026.md", + "title": "TechPulse Predictions for 2026: Ten Bets on Developer Technology", + "section-id": null, + "keywords": "2026 predictions, developer technology, AI tools, WebAssembly, open source, programming trends", + "description": "We make ten specific predictions for developer technology in 2026, and look back honestly at how our 2025 predictions fared.", + "author": "Clara Winthorpe", + "date": "2025-12-20", + "datetime": "2025-12-20 10:00", + "language": "en", + "body": "2025 retrospective: 6 correct, 2 wrong, 2 partial. 2026 predictions: WebAssembly in top-20 enterprise, AI coding assistant data breach, Rust crosses 25% adoption, maintainer certification system, developer tools IPO, AI standard library function generator, 'vibe coding' becomes negative signal, Linux Foundation maintenance programme, macOS ARM enterprise port, developer satisfaction declines despite usage increase." + }, + { + "file": "posts/2026-01-15-wasm-server.md", + "title": "Server-Side WebAssembly Is Finally Ready for Production", + "section-id": null, + "keywords": "WebAssembly, WASM server, Spin framework, WASI P2, production, serverless, Fermyon", + "description": "After years of 'almost there,' server-side WebAssembly has reached production readiness. Benchmark data, real case studies, and honest remaining rough edges.", + "author": "Raj Patel", + "date": "2026-01-15", + "datetime": "2026-01-15 09:00", + "language": "en", + "body": "Spin v3 and WASI P2 reach production readiness. Benchmarks: 1.2ms cold start, 87K req/s, 12MB memory. Three production case studies: plugin sandbox with security isolation, media company edge API with 2ms cold start, security scanning tool with concurrent plugin isolation. Rough edges: Rust-centric toolchain, harder debugging, smaller library ecosystem." + }, + { + "file": "posts/2026-02-28-open-source-llm-2026.md", + "title": "The State of Open Source LLMs: A 2026 Benchmark", + "section-id": null, + "keywords": "open source LLMs, LLM benchmarks, Llama, Mistral, AI models 2026, inference, enterprise AI", + "description": "We benchmarked 12 open-weight language models across reasoning, generation, cost, and deployment complexity.", + "author": "Raj Patel", + "date": "2026-02-28", + "datetime": "2026-02-28 14:00", + "language": "en", + "body": "12 models benchmarked. Top reasoning: Qwen 2.5 72B (74.1% MATH-500), Llama 3.3 70B (73.2%), DeepSeek R2 Distill (71.8% at 7B params). Cost: Llama 3.3 8B $0.08/M tokens vs GPT-4o $5-10/M. 7-70x cost advantage for self-hosted. Enterprise recommendation: Llama 3.3 70B or Qwen 2.5 72B for high-volume domain tasks. Frontier models still lead on hardest reasoning." + }, + { + "file": "posts/2026-03-20-platform-consolidation.md", + "title": "Developer Platform Consolidation: The End of the Microtools Era?", + "section-id": null, + "keywords": "developer platforms, GitHub, GitLab, Vercel, AWS, platform consolidation, developer tools market", + "description": "Survey data shows accelerating consolidation in developer tooling. Who is winning, who is losing, and what developers actually want.", + "author": "Maya Osei", + "date": "2026-03-20", + "datetime": "2026-03-20 10:30", + "language": "en", + "body": "GitHub 79% share (down from 86% in 2023). GitLab 14%, Gitea/Forgejo 5%. Vercel 28% for web apps but pricing concerns at scale. Developers prioritise: reliability (81%), pricing predictability (73%), DX quality (69%). Winners: GitHub, AWS, GitLab, Datadog. Losers: independent CI/CD services, niche security scanners." + }, + { + "file": "posts/2026-05-01-developer-survey-2026.md", + "title": "TechPulse Developer Survey 2026: AI Has Won, But Developers Have Mixed Feelings", + "section-id": null, + "keywords": "developer survey 2026, AI tools, deskilling, developer productivity, programming languages, AI adoption", + "description": "Our 2026 survey of 4,200 developers shows 78% use AI tools daily โ€” but concerns about deskilling, quality, and dependency are louder than ever.", + "author": "Maya Osei", + "date": "2026-05-01", + "datetime": "2026-05-01 09:00", + "language": "en", + "body": "4,200 respondents, 73 countries. 78% use AI tools daily (91% of 0-5 year developers, 61% of 16+ year developers). Cursor 29% market share vs Copilot 38%. 45% agree AI tools reduced ability to work without them. 'Very satisfied' dropped from 38% to 31%. Salary: 29% feel AI increases leverage, 28% feel it decreases it. Burnout 39%, job security concern entered top 5 stressors." + } +] diff --git a/sample-sites/techpulse/theme.yml b/sample-sites/techpulse/theme.yml new file mode 100644 index 0000000..da43002 --- /dev/null +++ b/sample-sites/techpulse/theme.yml @@ -0,0 +1,46 @@ +# mdcms theme โ€” TechPulse +light: + accent: "#0066CC" + background: "#FFFFFF" + nav-background: "#F0F4F8" + text: "#1A202C" + text-muted: "#718096" + +dark: + accent: "#63B3ED" + background: "#0D1117" + nav-background: "#161B22" + text: "#E2E8F0" + text-muted: "#A0AEC0" + +colours-semantic: + info: "#0066CC" + warning: "#D97706" + success: "#16A34A" + error: "#DC2626" + +callouts: + info: + icon: info + primary-colour: "#0066CC" + background-colour: "#0066CC" + warning: + icon: warning + primary-colour: "#D97706" + background-colour: "#D97706" + success: + icon: success + primary-colour: "#16A34A" + background-colour: "#16A34A" + error: + icon: error + primary-colour: "#DC2626" + background-colour: "#DC2626" + +font-body: "bunny:Inter:400" +font-heading: "bunny:Inter:700" +font-size: 1.0 +line-height: 1.7 + +main-width: 78em +nav-width: 16em diff --git a/sample-sites/velox-docs/assets/images/architecture.jpg b/sample-sites/velox-docs/assets/images/architecture.jpg new file mode 100644 index 0000000..6739f51 Binary files /dev/null and b/sample-sites/velox-docs/assets/images/architecture.jpg differ diff --git a/sample-sites/velox-docs/assets/images/hero.jpg b/sample-sites/velox-docs/assets/images/hero.jpg new file mode 100644 index 0000000..7541251 Binary files /dev/null and b/sample-sites/velox-docs/assets/images/hero.jpg differ diff --git a/sample-sites/velox-docs/assets/images/performance.jpg b/sample-sites/velox-docs/assets/images/performance.jpg new file mode 100644 index 0000000..36295a2 Binary files /dev/null and b/sample-sites/velox-docs/assets/images/performance.jpg differ diff --git a/sample-sites/velox-docs/config.yml b/sample-sites/velox-docs/config.yml new file mode 100644 index 0000000..5593571 --- /dev/null +++ b/sample-sites/velox-docs/config.yml @@ -0,0 +1,7 @@ +# mdcms v0.3 | DO NOT REMOVE THIS COMMENT +sitename: Velox Framework +sitedescription: The high-performance TypeScript web framework +navigation: sidebar +nav-position: left +search: true +footer: "ยฉ 2026 Velox Contributors. Licensed under MIT." diff --git a/sample-sites/velox-docs/index.html b/sample-sites/velox-docs/index.html new file mode 100644 index 0000000..1466873 --- /dev/null +++ b/sample-sites/velox-docs/index.html @@ -0,0 +1,2784 @@ + + + + + + + +MD-CMS + + + + + + + + + + + + + + +
+ + + + + + diff --git a/sample-sites/velox-docs/nav.yml b/sample-sites/velox-docs/nav.yml new file mode 100644 index 0000000..5bb125c --- /dev/null +++ b/sample-sites/velox-docs/nav.yml @@ -0,0 +1,227 @@ +# nav.yml โ€” generated by mdcms.py +sections: + - code: getting-started + defaultname: Getting Started + sort: 100 + pagesvisibility: visible + + - code: core-concepts + defaultname: Core Concepts + sort: 200 + pagesvisibility: visible + + - code: api-reference + defaultname: API Reference + sort: 300 + pagesvisibility: visible + + - code: guides + defaultname: Guides + sort: 400 + pagesvisibility: visible + + - code: deployment + defaultname: Deployment + sort: 500 + pagesvisibility: visible + +pages: + - file: pages/index.md + title: Introduction + section-id: getting-started + sort: 100 + variants: [en] + titles: + en: Introduction + + - file: pages/installation.md + title: Installation + section-id: getting-started + sort: 110 + variants: [en] + titles: + en: Installation + + - file: pages/quick-start.md + title: Quick Start + section-id: getting-started + sort: 120 + variants: [en] + titles: + en: Quick Start + + - file: pages/project-structure.md + title: Project Structure + section-id: getting-started + sort: 130 + variants: [en] + titles: + en: Project Structure + + - file: pages/configuration.md + title: Configuration + section-id: getting-started + sort: 140 + variants: [en] + titles: + en: Configuration + + - file: pages/routing.md + title: Routing + section-id: core-concepts + sort: 100 + variants: [en] + titles: + en: Routing + + - file: pages/components.md + title: Components + section-id: core-concepts + sort: 110 + variants: [en] + titles: + en: Components + + - file: pages/state-management.md + title: State Management + section-id: core-concepts + sort: 120 + variants: [en] + titles: + en: State Management + + - file: pages/data-fetching.md + title: Data Fetching + section-id: core-concepts + sort: 130 + variants: [en] + titles: + en: Data Fetching + + - file: pages/middleware.md + title: Middleware + section-id: core-concepts + sort: 140 + variants: [en] + titles: + en: Middleware + + - file: pages/layouts.md + title: Layouts + section-id: core-concepts + sort: 150 + variants: [en] + titles: + en: Layouts + + - file: pages/api-router.md + title: Router API + section-id: api-reference + sort: 100 + variants: [en] + titles: + en: Router API + + - file: pages/api-components.md + title: Component API + section-id: api-reference + sort: 110 + variants: [en] + titles: + en: Component API + + - file: pages/api-server.md + title: Server API + section-id: api-reference + sort: 120 + variants: [en] + titles: + en: Server API + + - file: pages/api-config.md + title: Config API + section-id: api-reference + sort: 130 + variants: [en] + titles: + en: Config API + + - file: pages/api-hooks.md + title: Hooks API + section-id: api-reference + sort: 140 + variants: [en] + titles: + en: Hooks API + + - file: pages/guide-auth.md + title: Authentication + section-id: guides + sort: 100 + variants: [en] + titles: + en: Authentication + + - file: pages/guide-database.md + title: Database Integration + section-id: guides + sort: 110 + variants: [en] + titles: + en: Database Integration + + - file: pages/guide-testing.md + title: Testing + section-id: guides + sort: 120 + variants: [en] + titles: + en: Testing + + - file: pages/guide-i18n.md + title: Internationalisation + section-id: guides + sort: 130 + variants: [en] + titles: + en: Internationalisation + + - file: pages/guide-performance.md + title: Performance + section-id: guides + sort: 140 + variants: [en] + titles: + en: Performance + + - file: pages/deploy-vercel.md + title: Deploy to Vercel + section-id: deployment + sort: 100 + variants: [en] + titles: + en: Deploy to Vercel + + - file: pages/deploy-cloudflare.md + title: Deploy to Cloudflare + section-id: deployment + sort: 110 + variants: [en] + titles: + en: Deploy to Cloudflare + + - file: pages/deploy-docker.md + title: Docker + section-id: deployment + sort: 120 + variants: [en] + titles: + en: Docker + + - file: pages/deploy-self-hosted.md + title: Self-Hosted + section-id: deployment + sort: 130 + variants: [en] + titles: + en: Self-Hosted diff --git a/sample-sites/velox-docs/pages/api-components.md b/sample-sites/velox-docs/pages/api-components.md new file mode 100644 index 0000000..d58659b --- /dev/null +++ b/sample-sites/velox-docs/pages/api-components.md @@ -0,0 +1,270 @@ +--- +title: Component API +sort: 110 +section-id: api-reference +keywords: defineComponent, ref, computed, watch, component API, reactive +description: Complete reference for the Velox component API โ€” defineComponent, ref, computed, and watch +language: en +--- + +# Component API + +This page documents the core component primitives exported from `velox/client`. + +## `signal(initialValue)` + +Creates a reactive signal โ€” a value container that triggers DOM updates when changed. + +```typescript +import { signal } from 'velox/client'; + +const count = signal(0); +count.value; // read: 0 +count.value = 5; // write: triggers reactive updates +count.peek(); // read without tracking (no reactive subscription) +``` + +### Type Signature + +```typescript +function signal(initialValue: T): Signal; + +interface Signal { + value: T; + peek(): T; + subscribe(listener: (value: T) => void): () => void; +} +``` + +## `computed(fn)` + +Creates a derived reactive value. Re-evaluates lazily when accessed and a dependency has changed. + +```typescript +import { signal, computed } from 'velox/client'; + +const price = signal(100); +const taxRate = signal(0.2); +const total = computed(() => price.value * (1 + taxRate.value)); + +total.value; // 120 +price.value = 200; +total.value; // 240 +``` + +Computed signals are read-only โ€” attempting to set `.value` throws. + +## `effect(fn)` + +Registers a reactive side effect. Runs immediately, then again whenever its signal dependencies change. + +```typescript +import { signal, effect } from 'velox/client'; + +const query = signal(''); + +const dispose = effect(() => { + console.log('query =', query.value); + // runs now, then on every change to query.value +}); + +// Clean up manually (effects inside components clean up on unmount): +dispose(); +``` + +Return a cleanup function from the effect to run before the next execution or on disposal: + +```typescript +effect(() => { + const controller = new AbortController(); + fetchData(query.value, controller.signal); + return () => controller.abort(); +}); +``` + +## `batch(fn)` + +Groups multiple signal writes into a single reactive update pass: + +```typescript +import { batch } from 'velox/client'; + +batch(() => { + a.value = 1; + b.value = 2; + c.value = 3; + // Only one round of re-renders happens +}); +``` + +## `untrack(fn)` + +Execute a function without tracking its signal reads as dependencies: + +```typescript +import { signal, effect, untrack } from 'velox/client'; + +const a = signal(1); +const b = signal(2); + +effect(() => { + // Only subscribes to `a`, not `b` + const result = a.value + untrack(() => b.value); + console.log(result); +}); +``` + +## Lifecycle Hooks + +Lifecycle hooks must be called synchronously during component initialisation (similar to React rules of hooks, but without the runtime check overhead). + +### `onMount(fn)` + +Called after the component's DOM is inserted into the document: + +```typescript +import { onMount } from 'velox/client'; + +onMount(() => { + // safe to access DOM, start timers, etc. + const input = document.querySelector('#my-input') as HTMLInputElement; + input.focus(); +}); +``` + +### `onCleanup(fn)` + +Called before the component unmounts, or before an effect runs again. Use to cancel subscriptions, abort requests, and clear timers: + +```typescript +import { onMount, onCleanup } from 'velox/client'; + +onMount(() => { + const timer = setInterval(tick, 1000); + onCleanup(() => clearInterval(timer)); +}); +``` + +### `onDestroy(fn)` + +Like `onCleanup`, but only runs when the component is permanently destroyed (not before re-renders): + +```typescript +import { onDestroy } from 'velox/client'; + +onDestroy(() => { + analytics.trackLeave(route); +}); +``` + +## `createContext` / `useContext` + +Create a context for dependency injection without prop drilling: + +```typescript +import { createContext, useContext } from 'velox/client'; + +// Create context with a default value +const ThemeContext = createContext<'light' | 'dark'>('light'); + +// Provide a value to a subtree + + + + +// Consume in any descendant +function Button() { + const theme = useContext(ThemeContext); + return ; +} +``` + +## `ref()` + +Create a DOM element reference: + +```typescript +import { ref, onMount } from 'velox/client'; + +export default function TextInput() { + const inputRef = ref(); + + onMount(() => { + inputRef.current?.focus(); + }); + + return ; +} +``` + +## `defineComponent(options)` + +The explicit component definition API โ€” useful when you need named components for debugging or when defining components programmatically: + +```typescript +import { defineComponent, signal } from 'velox/client'; + +const Counter = defineComponent({ + name: 'Counter', + props: { + initialValue: { type: Number, default: 0 }, + step: { type: Number, default: 1 }, + }, + setup(props) { + const count = signal(props.initialValue); + const increment = () => { count.value += props.step; }; + const decrement = () => { count.value -= props.step; }; + + return { count, increment, decrement }; + }, + render({ count, increment, decrement }) { + return ( +
+ + {count} + +
+ ); + }, +}); + +export default Counter; +``` + +## `lazy(loader)` + +Lazily load a component โ€” its code is only downloaded when the component is first rendered: + +```typescript +import { lazy } from 'velox/client'; + +const HeavyChart = lazy(() => import('./HeavyChart.tsx')); + +// Use in JSX โ€” shows nothing while loading by default + + +// With a Suspense boundary for a loading state +import { Suspense } from 'velox'; + +}> + + +``` + +## `memo(component)` + +Wrap a component in `memo` to skip re-renders when props have not changed (shallow equality): + +```typescript +import { memo } from 'velox/client'; + +const ExpensiveList = memo(function ExpensiveList({ items }: { items: string[] }) { + return
    {items.map(i =>
  • {i}
    • )}
; +}); +``` + +Accepts a custom equality function as the second argument: + +```typescript +const MyComponent = memo(Component, (prev, next) => prev.id === next.id); +``` diff --git a/sample-sites/velox-docs/pages/api-config.md b/sample-sites/velox-docs/pages/api-config.md new file mode 100644 index 0000000..718cfa8 --- /dev/null +++ b/sample-sites/velox-docs/pages/api-config.md @@ -0,0 +1,276 @@ +--- +title: Config API +sort: 130 +section-id: api-reference +keywords: config API, velox.config.ts, defineConfig, types, TypeScript reference +description: Full TypeScript type reference for velox.config.ts and all configuration options +language: en +--- + +# Config API + +This page provides the complete TypeScript type reference for `velox.config.ts`. All types are exported from the `velox` package. + +## `defineConfig(config)` + +The primary export. Accepts a `VeloxConfig` object and returns it with full type checking: + +```typescript +import { defineConfig } from 'velox'; + +export default defineConfig({ + // VeloxConfig object +}); +``` + +You can also pass a function for dynamic configuration: + +```typescript +export default defineConfig(async (env) => { + const secrets = await loadSecrets(); + return { + app: { + name: 'My App', + baseUrl: secrets.BASE_URL, + }, + }; +}); +``` + +## `VeloxConfig` + +```typescript +interface VeloxConfig { + app?: AppConfig; + server?: ServerConfig; + build?: BuildConfig; + routes?: RoutesConfig; + assets?: AssetsConfig; + css?: CSSConfig; + i18n?: I18nConfig; + middleware?: MiddlewareConfig[]; + plugins?: VeloxPlugin[]; + experimental?: ExperimentalConfig; + errorHandler?: ErrorHandler; +} +``` + +## `AppConfig` + +```typescript +interface AppConfig { + /** Application display name. Used in page titles and error pages. */ + name: string; + + /** Canonical base URL. Required in production. Example: 'https://example.com' */ + baseUrl: string; + + /** Default locale for i18n. Default: 'en' */ + defaultLocale?: string; + + /** Whether to append trailing slashes to all routes. Default: false */ + trailingSlash?: boolean; + + /** Custom 404 page route. Default: '_error' */ + notFoundPage?: string; +} +``` + +## `ServerConfig` + +```typescript +interface ServerConfig { + /** TCP port. Default: 3700 */ + port?: number; + + /** Bind hostname. Default: 'localhost' */ + host?: string; + + /** HTTPS configuration for the development server */ + https?: { + cert: string; // path to PEM certificate + key: string; // path to PEM private key + }; + + /** CORS policy */ + cors?: { + origin: string | string[] | ((origin: string) => boolean); + credentials?: boolean; + methods?: string[]; + allowedHeaders?: string[]; + exposedHeaders?: string[]; + maxAge?: number; + }; + + /** Compression. Default: true in production */ + compress?: boolean; + + /** Trust proxy headers (X-Forwarded-For, etc.). Default: false */ + trustProxy?: boolean | number; +} +``` + +## `BuildConfig` + +```typescript +interface BuildConfig { + /** + * Deployment target. + * - 'node': Outputs a Node.js server (default) + * - 'edge': Outputs a Web-API-compatible edge bundle + * - 'static': Full static export โ€” no server required + */ + target: 'node' | 'edge' | 'static'; + + /** Output directory. Default: '.velox/output' */ + outDir?: string; + + /** Source map generation. Default: false in production */ + sourcemap?: boolean | 'external' | 'inline'; + + /** Minify output. Default: true in production */ + minify?: boolean; + + /** Enable code splitting. Default: true */ + splitting?: boolean; + + /** Emit a bundle analysis HTML report */ + analyze?: boolean; + + /** + * Routes to explicitly pre-render as static HTML. + * Supports globs. e.g. ['/blog/*', '/about'] + */ + prerender?: string[]; + + /** External dependencies (not bundled). */ + external?: string[]; + + /** Environment variables to inline into the client bundle */ + define?: Record; +} +``` + +## `RoutesConfig` + +```typescript +interface RoutesConfig { + /** Routes directory relative to project root. Default: 'routes' */ + dir?: string; + + /** File extensions treated as routes. Default: ['.velox', '.tsx'] */ + extensions?: string[]; + + /** Glob patterns to exclude from routing */ + exclude?: string[]; + + /** Prefix all generated routes with this base path */ + base?: string; +} +``` + +## `AssetsConfig` + +```typescript +interface AssetsConfig { + /** Static assets directory. Default: 'public' */ + publicDir?: string; + + imageOptimisation?: { + enabled: boolean; + /** Output formats to generate. Default: ['webp'] */ + formats?: ('webp' | 'avif' | 'jpeg' | 'png')[]; + /** JPEG/WebP/AVIF quality 1โ€“100. Default: 80 */ + quality?: number; + /** Maximum width in pixels before downscaling */ + maxWidth?: number; + }; + + fonts?: { + /** Emit for fonts. Default: true */ + preload?: boolean; + /** Unicode range subsets. Example: ['latin', 'latin-ext'] */ + subsets?: string[]; + }; +} +``` + +## `I18nConfig` + +```typescript +interface I18nConfig { + /** Supported locale codes. Example: ['en', 'fr', 'de'] */ + locales: string[]; + + /** Default locale. Default: 'en' */ + defaultLocale: string; + + /** Strategy for locale in URL. Default: 'prefix-except-default' */ + routing?: 'prefix' | 'prefix-except-default' | 'domain'; + + /** Domain mapping for 'domain' routing strategy */ + domains?: Record; + + /** Path to translation files. Default: 'messages' */ + messagesDir?: string; +} +``` + +## `MiddlewareConfig` + +```typescript +interface MiddlewareConfig { + /** Route path glob to match. Use '*' for global middleware */ + path: string; + + /** Path to the middleware module (relative to project root) */ + handler: string; + + /** Execution order (lower runs first). Default: 0 */ + order?: number; +} +``` + +## `ExperimentalConfig` + +```typescript +interface ExperimentalConfig { + /** Enable CSS View Transitions for route changes */ + viewTransitions?: boolean; + + /** Enable fine-grained island hydration */ + partialHydration?: boolean; + + /** Enable compatibility shim for React components */ + reactCompat?: boolean; + + /** Enable server actions (form + RPC) */ + serverActions?: boolean; +} +``` + +## Plugin API + +```typescript +interface VeloxPlugin { + name: string; + setup?(build: VeloxBuild): void | Promise; + transform?(code: string, id: string): string | { code: string; map?: string } | null; + resolveId?(id: string, importer?: string): string | null; + load?(id: string): string | null; +} +``` + +Create a plugin: + +```typescript +function myPlugin(): VeloxPlugin { + return { + name: 'my-plugin', + transform(code, id) { + if (!id.endsWith('.myext')) return null; + return { code: transformMyExtension(code) }; + }, + }; +} +``` diff --git a/sample-sites/velox-docs/pages/api-hooks.md b/sample-sites/velox-docs/pages/api-hooks.md new file mode 100644 index 0000000..79ce840 --- /dev/null +++ b/sample-sites/velox-docs/pages/api-hooks.md @@ -0,0 +1,222 @@ +--- +title: Hooks API +sort: 140 +section-id: api-reference +keywords: hooks, useRequest, useSession, useCookies, useEnv, server hooks +description: Reference for Velox server-side hooks โ€” useRequest, useSession, useCookies, and useEnv +language: en +--- + +# Hooks API + +Velox provides server-side hooks for accessing request context, sessions, cookies, and environment variables within route server blocks and middleware. These hooks are only available in server-side contexts. + +## `useRequest()` + +Returns the current `VeloxRequest` object: + +```typescript +import { useRequest } from 'velox/server'; + +const request = useRequest(); + +const method = request.method; +const pathname = new URL(request.url).pathname; +const userAgent = request.headers.get('User-Agent'); +``` + +This is equivalent to the `request` variable that is automatically available in server blocks, but `useRequest()` is useful in helper functions that are called from a server block without threading `request` through manually. + +### Full Request Object Reference + +```typescript +interface VeloxRequest extends Request { + params: Record; // route dynamic params + query: URLSearchParams; // parsed query string + cookies: RequestCookies; // parsed cookies + context: Map; // middleware-set values + ip: string | null; // client IP address + geo: GeoInfo | null; // geographic info (if available) +} +``` + +## `useSession()` + +Reads and writes the server-managed session. Sessions are stored server-side (in memory, Redis, or a database depending on your `session.store` configuration) and identified by a signed cookie. + +```typescript +import { useSession } from 'velox/server'; + +const session = await useSession<{ userId: string; role: string }>(); + +// Read +const userId = session.data.userId; +const role = session.data.role; + +// Write โ€” persists changes to the session store +await session.set('userId', '123'); +await session.set('role', 'admin'); + +// Update multiple at once +await session.update({ userId: '123', role: 'admin' }); + +// Destroy the session (logout) +await session.destroy(); + +// Regenerate session ID (after privilege change โ€” prevents fixation) +await session.regenerate(); +``` + +### Session Configuration + +Configure the session store in `velox.config.ts`: + +```typescript +import { defineConfig } from 'velox'; +import { RedisSessionStore } from '@velox/session-redis'; + +export default defineConfig({ + session: { + secret: process.env.SESSION_SECRET!, + cookieName: 'velox.session', + maxAge: 60 * 60 * 24 * 7, // 7 days + httpOnly: true, + secure: process.env.NODE_ENV === 'production', + sameSite: 'lax', + store: new RedisSessionStore({ + url: process.env.REDIS_URL!, + }), + }, +}); +``` + +## `useCookies()` + +Read and write cookies. Returns a `CookieJar` object: + +```typescript +import { useCookies } from 'velox/server'; + +const cookies = useCookies(); + +// Read +const theme = cookies.get('theme')?.value ?? 'light'; +const hasConsented = cookies.get('consent')?.value === 'true'; + +// Write (adds Set-Cookie header to the response) +cookies.set('theme', 'dark', { + path: '/', + maxAge: 365 * 24 * 60 * 60, // 1 year + sameSite: 'lax', +}); + +// Delete +cookies.delete('old-cookie', { path: '/' }); +``` + +### Cookie Options + +| Option | Type | Description | +|--------|------|-------------| +| `domain` | `string` | Cookie domain | +| `path` | `string` | Cookie path. Default: `/` | +| `maxAge` | `number` | Expiry in seconds | +| `expires` | `Date` | Expiry as a date | +| `httpOnly` | `boolean` | Prevent JavaScript access | +| `secure` | `boolean` | HTTPS only | +| `sameSite` | `'strict' \| 'lax' \| 'none'` | SameSite policy | + +## `useEnv(key, defaultValue?)` + +Type-safe environment variable access: + +```typescript +import { useEnv } from 'velox/server'; + +const dbUrl = useEnv('DATABASE_URL'); // throws if not set +const port = useEnv('PORT', '3700'); // returns default if not set +const debug = useEnv.boolean('DEBUG', false); // parse as boolean +const timeout = useEnv.number('TIMEOUT', 30); // parse as number +``` + +### `useEnv` Methods + +| Method | Description | +|--------|-------------| +| `useEnv(key)` | Return string value, throw if missing | +| `useEnv(key, default)` | Return string value or default | +| `useEnv.boolean(key, default?)` | Parse as boolean (`'true'` / `'1'` = `true`) | +| `useEnv.number(key, default?)` | Parse as float | +| `useEnv.json(key, default?)` | Parse as JSON | +| `useEnv.url(key, default?)` | Validate and return as URL string | + +## `useHeaders()` + +Access and modify response headers from within a server block or middleware: + +```typescript +import { useHeaders } from 'velox/server'; + +const headers = useHeaders(); + +// Read request headers +const accept = headers.request.get('Accept'); + +// Set response headers +headers.response.set('Cache-Control', 'public, max-age=3600'); +headers.response.set('X-Custom-Header', 'value'); +``` + +## `useLocale()` + +Returns the current request locale (resolved by the i18n middleware): + +```typescript +import { useLocale } from 'velox/server'; + +const { locale, locales, defaultLocale } = useLocale(); + +// locale: 'fr' (current request locale) +// locales: ['en', 'fr', 'de'] (all supported locales) +// defaultLocale: 'en' +``` + +## `useAuth()` + +A convenience hook that reads the current user from the session. Returns `null` if not authenticated: + +```typescript +import { useAuth } from '$lib/auth'; // project-defined hook + +const user = await useAuth(); + +if (!user) { + throw redirect('/login'); +} + +// user: { id: string, email: string, role: string } +``` + +The `useAuth` hook is not provided by Velox itself โ€” you define it in your project using `useSession` and your own user model. The [Authentication guide](guide-auth.md) shows a complete implementation. + +## `useCache()` + +Interact with Velox's built-in server-side cache: + +```typescript +import { useCache } from 'velox/server'; + +const cache = useCache(); + +// Get from cache +const cached = await cache.get('users:all'); +if (cached) return cached; + +// Set with TTL (seconds) +const users = await db.users.findMany(); +await cache.set('users:all', users, { ttl: 300 }); + +// Invalidate +await cache.delete('users:all'); +await cache.deletePattern('users:*'); +``` diff --git a/sample-sites/velox-docs/pages/api-router.md b/sample-sites/velox-docs/pages/api-router.md new file mode 100644 index 0000000..8911d75 --- /dev/null +++ b/sample-sites/velox-docs/pages/api-router.md @@ -0,0 +1,227 @@ +--- +title: Router API +sort: 100 +section-id: api-reference +keywords: router, useRouter, navigate, Link, createRouter, programmatic navigation +description: Complete API reference for the Velox router โ€” createRouter, useRouter, navigate, and Link +language: en +--- + +# Router API + +The Velox router provides both declarative and programmatic navigation. This page documents the full public API surface of `@velox/router`. + +## `useRouter()` + +The `useRouter` hook returns the current router instance. It is available inside any client-side component: + +```typescript +import { useRouter } from 'velox/client'; + +const router = useRouter(); +``` + +### Router Instance Properties + +| Property | Type | Description | +|----------|------|-------------| +| `pathname` | `string` | Current URL pathname, e.g. `/blog/my-post` | +| `search` | `string` | Current query string including `?`, e.g. `?page=2` | +| `hash` | `string` | Current hash fragment including `#` | +| `params` | `Record` | Dynamic route parameters | +| `query` | `URLSearchParams` | Parsed query parameters | +| `state` | `unknown` | History state object (if provided to `navigate`) | + +### Router Instance Methods + +#### `navigate(href, options?)` + +Performs client-side navigation to the given href: + +```typescript +router.navigate('/dashboard'); + +// With options: +router.navigate('/profile/edit', { + replace: true, // replace current history entry + state: { from: '/dashboard' }, // pass arbitrary state + scroll: false, // don't scroll to top after navigation +}); +``` + +#### `back()` / `forward()` + +Navigate through the browser history stack: + +```typescript +router.back(); +router.forward(); +``` + +#### `prefetch(href)` + +Manually prefetch a route (downloads the JS bundle and optionally data): + +```typescript +router.prefetch('/heavy-page'); +``` + +#### `refresh()` + +Re-run the current route's server block and update the page without a full navigation: + +```typescript +router.refresh(); +``` + +## `` Component + +The `` component renders an accessible `` element with client-side navigation and built-in prefetching: + +```tsx +import { Link } from 'velox/client'; + +About +``` + +### Props + +| Prop | Type | Default | Description | +|------|------|---------|-------------| +| `href` | `string` | required | Destination URL | +| `prefetch` | `'hover' \| 'viewport' \| false` | `'hover'` | Prefetch strategy | +| `replace` | `boolean` | `false` | Replace history entry instead of pushing | +| `scroll` | `boolean` | `true` | Scroll to top after navigation | +| `state` | `unknown` | `undefined` | History state object | +| `activeClass` | `string` | `'active'` | CSS class applied when href matches current pathname | +| `exactActiveClass` | `string` | `'exact-active'` | CSS class applied only on exact pathname match | + +```tsx + + Blog + +``` + +## `createRouter()` + +For testing or server-side use, create a router instance manually: + +```typescript +import { createRouter } from 'velox/router'; + +const router = createRouter({ + base: '/app', // base path prefix + history: 'hash', // 'browser' (default) | 'hash' | 'memory' + scrollRestoration: 'auto', // 'auto' | 'manual' +}); +``` + +## `useParams()` + +Access current route parameters in any component: + +```typescript +import { useParams } from 'velox/client'; + +const { slug, category } = useParams<{ slug: string; category: string }>(); +``` + +## `useSearchParams()` + +Read and update the URL search parameters reactively: + +```typescript +import { useSearchParams } from 'velox/client'; + +const [searchParams, setSearchParams] = useSearchParams(); + +// Read +const page = searchParams.get('page') ?? '1'; + +// Update (triggers navigation without full page reload) +setSearchParams({ page: String(currentPage + 1) }); + +// Merge with existing params +setSearchParams(prev => { + prev.set('sort', 'date'); + return prev; +}); +``` + +## `usePathname()` + +Reactively access the current pathname: + +```typescript +import { usePathname } from 'velox/client'; + +const pathname = usePathname(); +// pathname is a Signal +``` + +## `useNavigate()` + +A lightweight hook that returns only the `navigate` function, without the full router object: + +```typescript +import { useNavigate } from 'velox/client'; + +const navigate = useNavigate(); +navigate('/login'); +``` + +## Navigation Events + +Subscribe to router lifecycle events: + +```typescript +import { useRouter } from 'velox/client'; + +const router = useRouter(); + +const unsubscribe = router.on('beforeNavigate', ({ to, from, cancel }) => { + if (hasUnsavedChanges && !confirm('Leave without saving?')) { + cancel(); + } +}); + +// Clean up +onCleanup(unsubscribe); +``` + +Available events: `beforeNavigate`, `afterNavigate`, `navigationError`. + +## `redirect()` (Server) + +Used inside server blocks and API route handlers: + +```typescript +import { redirect } from 'velox/server'; + +// Temporary redirect (302) +throw redirect('/login'); + +// Permanent redirect (301) +throw redirect('/new-url', 301); + +// With custom headers +throw redirect('/dashboard', 302, { + 'Set-Cookie': 'session=...; Path=/', +}); +``` + +## `notFound()` (Server) + +Trigger the nearest `_error.velox` with a 404 status: + +```typescript +import { notFound } from 'velox/server'; + +const post = await db.posts.find(id); +if (!post) throw notFound(); +``` diff --git a/sample-sites/velox-docs/pages/api-server.md b/sample-sites/velox-docs/pages/api-server.md new file mode 100644 index 0000000..3f46293 --- /dev/null +++ b/sample-sites/velox-docs/pages/api-server.md @@ -0,0 +1,243 @@ +--- +title: Server API +sort: 120 +section-id: api-reference +keywords: server API, createServer, defineHandler, Response helpers, server-side +description: Reference for Velox server-side APIs โ€” createServer, defineHandler, and Response helpers +language: en +--- + +# Server API + +The Velox server API provides utilities for API route handlers, middleware, and server-side logic. All exports documented here come from `velox/server`. + +## `defineHandler(fn)` + +Wraps an API route handler with type inference and error handling: + +```typescript +import { defineHandler } from 'velox/server'; + +export const GET = defineHandler(async (request) => { + return Response.json({ status: 'ok' }); +}); +``` + +The handler receives a `VeloxRequest` object (an extension of the standard `Request`) and must return a `Response` or `Promise`. + +### `VeloxRequest` + +The enhanced request object passed to handlers: + +| Property | Type | Description | +|----------|------|-------------| +| `url` | `string` | Full request URL | +| `method` | `string` | HTTP method (uppercase) | +| `headers` | `Headers` | Request headers | +| `params` | `Record` | URL route parameters | +| `query` | `URLSearchParams` | Parsed query string | +| `cookies` | `RequestCookies` | Parsed cookies | +| `context` | `Map` | Values set by middleware | +| `json()` | `Promise` | Parse body as JSON | +| `text()` | `Promise` | Parse body as text | +| `formData()` | `Promise` | Parse body as form data | +| `arrayBuffer()` | `Promise` | Parse body as binary | + +## Response Helpers + +Velox exports a set of `Response` factory helpers for common responses: + +### `json(data, init?)` + +Return a JSON response with appropriate `Content-Type` header: + +```typescript +import { json } from 'velox/server'; + +return json({ users }, { status: 200 }); +return json({ error: 'Not found' }, { status: 404 }); +``` + +### `text(body, init?)` + +Return a plain text response: + +```typescript +import { text } from 'velox/server'; + +return text('Hello, World!'); +return text('Not Found', { status: 404 }); +``` + +### `html(body, init?)` + +Return an HTML response: + +```typescript +import { html } from 'velox/server'; + +return html('

Hello

'); +``` + +### `redirect(url, status?, headers?)` + +Return a redirect response: + +```typescript +import { redirect } from 'velox/server'; + +return redirect('/login', 302); +return redirect('/new-location', 301); +``` + +### `notFound(message?)` + +Return a 404 response: + +```typescript +import { notFound } from 'velox/server'; + +return notFound('User not found'); +// Returns: Response with status 404 and JSON body +``` + +### `unauthorized(message?)` + +Return a 401 response: + +```typescript +import { unauthorized } from 'velox/server'; + +return unauthorized('Please log in'); +``` + +### `forbidden(message?)` + +Return a 403 response: + +```typescript +import { forbidden } from 'velox/server'; + +return forbidden('You do not have access to this resource'); +``` + +### `badRequest(message?, details?)` + +Return a 400 response with optional validation details: + +```typescript +import { badRequest } from 'velox/server'; + +return badRequest('Validation failed', { field: 'email', message: 'Invalid format' }); +``` + +### `stream(generator, init?)` + +Stream a response body using an async generator: + +```typescript +import { stream } from 'velox/server'; + +export const GET = defineHandler(async () => { + return stream(async function* () { + for await (const chunk of getLargeDataset()) { + yield JSON.stringify(chunk) + '\n'; + } + }); +}); +``` + +## `createServer(options)` + +Bootstrap a standalone Velox server programmatically (useful for testing): + +```typescript +import { createServer } from 'velox/server'; + +const server = await createServer({ + root: '/path/to/project', + port: 3700, + mode: 'development', +}); + +await server.start(); +console.log(`Server running on port ${server.port}`); + +// Later: +await server.stop(); +``` + +### `VeloxServer` Methods + +| Method | Description | +|--------|-------------| +| `start()` | Start the HTTP server | +| `stop()` | Gracefully shut down the server | +| `getUrl()` | Returns the server base URL as a string | +| `inject(request)` | Inject a synthetic request for testing without a network | + +## Cookies + +### Reading Cookies + +```typescript +const session = request.cookies.get('session'); +const userId = request.cookies.get('userId')?.value; +``` + +### Setting Cookies + +Use the `ResponseCookies` API on the response object, or the `setCookie` helper: + +```typescript +import { setCookie, deleteCookie } from 'velox/server'; + +const response = json({ ok: true }); +setCookie(response, 'session', token, { + httpOnly: true, + secure: true, + sameSite: 'lax', + path: '/', + maxAge: 60 * 60 * 24 * 7, // 7 days +}); + +return response; +``` + +### Deleting Cookies + +```typescript +const response = redirect('/login'); +deleteCookie(response, 'session'); +return response; +``` + +## `useEnv(key, defaultValue?)` + +Type-safe environment variable access with optional validation: + +```typescript +import { useEnv } from 'velox/server'; + +const dbUrl = useEnv('DATABASE_URL'); // throws if missing +const port = useEnv('PORT', '3700'); // returns defaultValue if missing +const apiKey = useEnv.required('SECRET_KEY'); // alias for useEnv(key) +``` + +## Error Handling + +Velox catches thrown errors in route handlers and middleware. Define a custom error handler in `velox.config.ts`: + +```typescript +import { defineConfig } from 'velox'; + +export default defineConfig({ + errorHandler: async (error, request) => { + if (error.status === 404) { + return json({ error: 'Not Found' }, { status: 404 }); + } + console.error(error); + return json({ error: 'Internal Server Error' }, { status: 500 }); + }, +}); +``` diff --git a/sample-sites/velox-docs/pages/components.md b/sample-sites/velox-docs/pages/components.md new file mode 100644 index 0000000..b30a8ac --- /dev/null +++ b/sample-sites/velox-docs/pages/components.md @@ -0,0 +1,253 @@ +--- +title: Components +sort: 110 +section-id: core-concepts +keywords: components, props, slots, lifecycle, tsx, velox components, islands +description: The Velox component model, including props, slots, lifecycle hooks, and the islands architecture +language: en +--- + +# Components + +Velox components are TypeScript/TSX files that describe a piece of UI. They are the fundamental building blocks of a Velox application โ€” reusable, composable, and by default rendered entirely on the server. + +## Defining a Component + +A basic Velox component is a TypeScript function that returns JSX: + +```tsx +// components/Greeting.tsx +interface GreetingProps { + name: string; + formal?: boolean; +} + +export default function Greeting({ name, formal = false }: GreetingProps) { + const salutation = formal ? 'Good day' : 'Hello'; + return

{salutation}, {name}!

; +} +``` + +Use it in a route or another component: + +```tsx +--- +import Greeting from '../components/Greeting.tsx'; +--- + +
+ + +
+``` + +## Props + +Props are typed using TypeScript interfaces or types. All props are validated at compile time โ€” no runtime prop checking overhead. + +```tsx +interface CardProps { + title: string; + description: string; + href?: string; + variant?: 'default' | 'featured' | 'compact'; + children?: VeloxNode; +} + +export default function Card({ + title, + description, + href, + variant = 'default', + children, +}: CardProps) { + return ( +
+

{href ? {title} : title}

+

{description}

+ {children &&
{children}
} +
+ ); +} +``` + +### Default Props + +Set default values directly in the destructuring parameter, as shown above. Velox does not use a separate `defaultProps` mechanism. + +### Required vs Optional Props + +By TypeScript convention, optional props are marked with `?`. Omitting a required prop is a compile-time error. + +## Slots + +The `children` prop is the default slot โ€” content placed between the opening and closing tags of a component: + +```tsx + +

This is the card body.

+ +``` + +### Named Slots + +For more complex component APIs, use named slot props: + +```tsx +interface ModalProps { + title: VeloxNode; + footer?: VeloxNode; + children: VeloxNode; +} + +export default function Modal({ title, footer, children }: ModalProps) { + return ( +
+
{title}
+
{children}
+ {footer &&
{footer}
} +
+ ); +} +``` + +Usage: + +```tsx +Confirm Delete} + footer={ + <> + + + + } +> +

Are you sure you want to delete this item?

+ +``` + +## Server vs Client Components + +By default, all Velox components run on the server. They have no JavaScript bundle size on the client and cannot use browser APIs or client-side reactivity. + +To make a component interactive on the client, use the `client:*` hydration directive when you include it in a route: + +```tsx +--- +import Counter from '../components/Counter.tsx'; +import LazyChart from '../components/LazyChart.tsx'; +--- + + + + + + + + + + + + +``` + +### Writing Client Components + +Client components can use signals, effects, and browser APIs: + +```tsx +import { signal, effect, onMount, onCleanup } from 'velox/client'; + +export default function LiveClock() { + const time = signal(new Date().toLocaleTimeString()); + + onMount(() => { + const interval = setInterval(() => { + time.value = new Date().toLocaleTimeString(); + }, 1000); + + onCleanup(() => clearInterval(interval)); + }); + + return

Current time: {time}

; +} +``` + +## Lifecycle Hooks + +Client-side components can use the following lifecycle hooks: + +| Hook | When it runs | +|------|-------------| +| `onMount(fn)` | After the component is first rendered and inserted into the DOM | +| `onUpdate(fn)` | After every re-render (reactive signal change) | +| `onCleanup(fn)` | Before the component unmounts or before the next `onUpdate` call | +| `onDestroy(fn)` | When the component is permanently unmounted | + +```tsx +import { signal, onMount, onUpdate, onCleanup } from 'velox/client'; + +export default function DataComponent() { + const data = signal([]); + const loading = signal(true); + + onMount(async () => { + const response = await fetch('/api/data'); + data.value = await response.json(); + loading.value = false; + }); + + onCleanup(() => { + // abort pending requests, clear timers, etc. + }); + + return ( +
+ {loading.value ?

Loading...

:
    {data.value.map(d =>
  • {d.name}
    • )}
} +
+ ); +} +``` + +## Component Composition Patterns + +### Higher-Order Components + +```tsx +function withAuth(Component: VeloxComponent) { + return function AuthGated(props: T) { + const user = useUser(); + if (!user) return ; + return ; + }; +} +``` + +### Render Props / Function-as-Children + +```tsx +interface FetcherProps { + url: string; + render: (data: T) => VeloxNode; +} + +export function Fetcher({ url, render }: FetcherProps) { + // ... fetch logic + return render(data); +} +``` + +## Component Library + +Velox ships an optional official component library `@velox/ui` with accessible, unstyled base components. Install it separately: + +```bash +npm install @velox/ui +``` + +```tsx +import { Button, Input, Dialog } from '@velox/ui'; +``` + +See the [Component API](api-components.md) reference for the full `defineComponent` API and advanced component patterns. diff --git a/sample-sites/velox-docs/pages/configuration.md b/sample-sites/velox-docs/pages/configuration.md new file mode 100644 index 0000000..97919fa --- /dev/null +++ b/sample-sites/velox-docs/pages/configuration.md @@ -0,0 +1,249 @@ +--- +title: Configuration +sort: 140 +section-id: getting-started +keywords: configuration, velox.config.ts, settings, options, defineConfig +description: Complete reference for velox.config.ts and all available configuration options +language: en +--- + +# Configuration + +Velox is configured through a single `velox.config.ts` file at the root of your project. This file is evaluated at build time (and at dev-server startup) to determine how Velox should compile and serve your application. + +## Creating the Config File + +If you used `create-velox` to scaffold your project, a `velox.config.ts` is generated for you. To create one manually: + +```typescript +import { defineConfig } from 'velox'; + +export default defineConfig({ + // options go here +}); +``` + +`defineConfig` is a helper that provides full TypeScript type inference over the configuration object โ€” use it rather than exporting a plain object. + +## Top-Level Options + +### `app` + +General application metadata. + +```typescript +app: { + name: string; // used in HTML and meta tags + baseUrl: string; // canonical base URL (e.g. https://example.com) + defaultLocale?: string; // default locale for i18n (default: 'en') + trailingSlash?: boolean; // append trailing slash to all routes (default: false) +} +``` + +Example: + +```typescript +app: { + name: 'My Velox App', + baseUrl: process.env.PUBLIC_BASE_URL ?? 'http://localhost:3700', + defaultLocale: 'en', + trailingSlash: false, +}, +``` + +### `server` + +Development and production server settings. + +```typescript +server: { + port?: number; // dev server port (default: 3700) + host?: string; // bind address (default: 'localhost') + https?: { // enable HTTPS for the dev server + cert: string; // path to certificate file + key: string; // path to key file + }; + cors?: { + origin: string | string[] | '*'; + credentials?: boolean; + methods?: string[]; + }; +} +``` + +### `build` + +Build system options. + +```typescript +build: { + target: 'node' | 'edge' | 'static'; // deployment target + outDir?: string; // output directory (default: '.velox/output') + sourcemap?: boolean | 'external'; // generate source maps + minify?: boolean; // minify output (default: true in production) + splitting?: boolean; // enable code splitting (default: true) + analyze?: boolean; // emit a bundle analysis report + prerender?: string[]; // explicit list of routes to pre-render +} +``` + +### `routes` + +Fine-grained routing options. + +```typescript +routes: { + dir?: string; // routes directory (default: 'routes') + extensions?: string[]; // file extensions treated as routes + // default: ['.velox', '.tsx', '.ts'] + exclude?: string[]; // glob patterns to exclude +} +``` + +### `assets` + +Asset handling and optimisation. + +```typescript +assets: { + publicDir?: string; // static assets directory (default: 'public') + imageOptimisation?: { + enabled: boolean; + formats?: ('webp' | 'avif')[]; + quality?: number; // 1โ€“100, default 80 + }; + fonts?: { + preload?: boolean; + subsets?: string[]; + }; +} +``` + +### `css` + +Stylesheet handling. + +```typescript +css: { + modules?: { + localsConvention?: 'camelCase' | 'camelCaseOnly' | 'dashes'; + generateScopedName?: string; + }; + preprocessors?: { + sass?: boolean; // enable Sass (requires @velox/sass) + less?: boolean; + stylus?: boolean; + }; + postcss?: { + plugins?: any[]; + }; +} +``` + +### `plugins` + +The plugin array lets you extend Velox with first-party and community plugins. + +```typescript +import { defineConfig } from 'velox'; +import { veloxMDX } from '@velox/mdx'; +import { veloxSass } from '@velox/sass'; +import { veloxPWA } from '@velox/pwa'; + +export default defineConfig({ + plugins: [ + veloxMDX(), + veloxSass(), + veloxPWA({ + name: 'My App', + themeColor: '#3a7bd5', + }), + ], +}); +``` + +### `experimental` + +Opt-in to experimental features that are not yet stable. + +```typescript +experimental: { + viewTransitions?: boolean; // CSS View Transitions API support + partialHydration?: boolean; // fine-grained component hydration + reactCompat?: boolean; // React component compatibility shim +} +``` + +## Environment-Specific Configuration + +You can provide environment-specific overrides: + +```typescript +import { defineConfig } from 'velox'; + +export default defineConfig({ + app: { + name: 'My App', + baseUrl: process.env.PUBLIC_BASE_URL!, + }, + server: { + port: parseInt(process.env.PORT ?? '3700'), + }, + build: { + target: process.env.VELOX_TARGET as 'node' | 'edge' ?? 'node', + sourcemap: process.env.NODE_ENV !== 'production', + }, +}); +``` + +## Per-Route Rendering Configuration + +You can override the rendering mode for individual routes from within the route file: + +```typescript +// routes/dashboard.velox server block +export const config = { + render: 'ssr', // 'ssr' | 'ssg' | 'isr' | 'csr' + revalidate: 60, // ISR: revalidate every 60 seconds + edge: true, // run on edge runtime +}; +``` + +## Full Example + +A production-ready `velox.config.ts` for a large application: + +```typescript +import { defineConfig } from 'velox'; +import { veloxMDX } from '@velox/mdx'; + +export default defineConfig({ + app: { + name: 'Acme Corp', + baseUrl: process.env.PUBLIC_BASE_URL!, + defaultLocale: 'en', + }, + server: { + port: 3700, + cors: { + origin: process.env.ALLOWED_ORIGINS?.split(',') ?? '*', + credentials: true, + }, + }, + build: { + target: 'edge', + sourcemap: 'external', + analyze: process.env.ANALYZE === '1', + }, + assets: { + imageOptimisation: { + enabled: true, + formats: ['webp', 'avif'], + quality: 85, + }, + }, + plugins: [veloxMDX()], +}); +``` + +For the complete type definitions, see the [Config API](api-config.md) reference. diff --git a/sample-sites/velox-docs/pages/data-fetching.md b/sample-sites/velox-docs/pages/data-fetching.md new file mode 100644 index 0000000..52a6ad4 --- /dev/null +++ b/sample-sites/velox-docs/pages/data-fetching.md @@ -0,0 +1,267 @@ +--- +title: Data Fetching +sort: 130 +section-id: core-concepts +keywords: data fetching, SSR, SSG, ISR, server-side rendering, static generation, fetch, async +description: How to fetch data in Velox using SSR, SSG, ISR, and client-side fetching patterns +language: en +--- + +# Data Fetching + +Velox provides multiple data-fetching patterns depending on when and how often your data changes. You can mix strategies freely across routes in the same project. + +## Server-Side Rendering (SSR) + +In SSR mode, data is fetched fresh on every request. Use `await` directly in the server block of a `.velox` route: + +```tsx +--- +// routes/blog/[slug].velox +import { db } from '$lib/db'; +import { NotFound } from 'velox/server'; + +const { slug } = params; +const post = await db.posts.findBySlug(slug); + +if (!post) throw new NotFound(); + +export const meta = { + title: post.title, + description: post.excerpt, +}; +export const config = { render: 'ssr' }; +--- + +<article> + <h1>{post.title}</h1> + <p class="byline">By {post.author} on {post.publishedAt}</p> + <div innerHTML={post.htmlContent} /> +</article> +``` + +The `render: 'ssr'` export is optional โ€” SSR is the default for routes that contain `await` expressions. + +### Request Context in SSR + +In SSR mode, the `request` object is available in the server block: + +```typescript +const authHeader = request.headers.get('Authorization'); +const userAgent = request.headers.get('User-Agent'); +const cookieHeader = request.headers.get('Cookie'); +``` + +## Static Site Generation (SSG) + +SSG routes are rendered once at build time. They are ideal for content that rarely changes. + +```tsx +--- +export const config = { render: 'ssg' }; + +const features = await fetch('https://api.example.com/features').then(r => r.json()); +--- + +<section> + <h1>Features</h1> + <ul> + {features.map(f => <li>{f.name}: {f.description}</li>)} + </ul> +</section> +``` + +### Dynamic SSG Routes + +For SSG routes with dynamic segments, export a `paths` function to tell Velox which parameter values to pre-render: + +```tsx +--- +import { db } from '$lib/db'; + +export const config = { render: 'ssg' }; + +// Called at build time to enumerate all slugs +export async function paths() { + const slugs = await db.posts.findManySlugs(); + return slugs.map(slug => ({ slug })); +} + +const { slug } = params; +const post = await db.posts.findBySlug(slug); +export const meta = { title: post.title }; +--- + +<article> + <h1>{post.title}</h1> + <div innerHTML={post.htmlContent} /> +</article> +``` + +## Incremental Static Regeneration (ISR) + +ISR generates pages statically but revalidates them in the background after a configurable interval: + +```tsx +--- +export const config = { + render: 'isr', + revalidate: 3600, // regenerate at most once per hour +}; + +const products = await db.products.findMany({ orderBy: 'created_at' }); +--- + +<section> + {products.map(p => <ProductCard product={p} />)} +</section> +``` + +On a cache miss (first request, or after `revalidate` seconds), Velox serves the stale page immediately while regenerating the fresh version in the background. The next request gets the fresh version. + +### Manual Revalidation + +Trigger revalidation programmatically (e.g., from a webhook): + +```typescript +// routes/api/revalidate+server.ts +import { revalidatePath, revalidateTag } from 'velox/server'; + +export const POST = defineHandler(async (req) => { + const { path, secret } = await req.json(); + + if (secret !== process.env.REVALIDATE_SECRET) { + return Response.json({ error: 'Unauthorized' }, { status: 401 }); + } + + await revalidatePath(path); + return Response.json({ revalidated: true }); +}); +``` + +Tag-based revalidation: + +```typescript +// When fetching, attach cache tags: +const data = await fetch('/api/products', { + next: { tags: ['products'] } +}); + +// Later, invalidate all pages that used the 'products' tag: +await revalidateTag('products'); +``` + +## Client-Side Fetching + +For data that must be fresh on the client (user-specific dashboards, real-time data), use client-side fetching in an interactive component: + +```tsx +import { signal, onMount } from 'velox/client'; + +export default function UserDashboard() { + const stats = signal<DashboardStats | null>(null); + const error = signal<string | null>(null); + const loading = signal(true); + + onMount(async () => { + try { + const res = await fetch('/api/dashboard/stats', { + credentials: 'include', + }); + if (!res.ok) throw new Error('Failed to fetch stats'); + stats.value = await res.json(); + } catch (e: any) { + error.value = e.message; + } finally { + loading.value = false; + } + }); + + return ( + <div> + {loading.value && <Spinner />} + {error.value && <ErrorMessage message={error.value} />} + {stats.value && <StatsGrid stats={stats.value} />} + </div> + ); +} +``` + +### Using `@velox/query` + +For production client-side data fetching, `@velox/query` handles caching, deduplication, background refetch, and stale-while-revalidate: + +```typescript +import { useQuery } from '@velox/query'; + +export default function ProductList() { + const { data, status, error, refetch } = useQuery({ + key: ['products'], + fetcher: () => fetch('/api/products').then(r => r.json()), + staleTime: 30_000, + refetchOnWindowFocus: true, + }); + + if (status === 'loading') return <Spinner />; + if (status === 'error') return <p>Error: {error.message}</p>; + + return ( + <ul> + {data.map(p => <li key={p.id}>{p.name}</li>)} + </ul> + ); +} +``` + +## Parallel Data Fetching + +Fetch multiple data sources in parallel using `Promise.all`: + +```tsx +--- +const [user, posts, categories] = await Promise.all([ + db.users.findById(userId), + db.posts.findByUser(userId), + db.categories.findAll(), +]); +--- +``` + +## Streaming + +For pages with slow data, use streaming to send the page shell immediately and stream in the slow parts: + +```tsx +--- +import { Suspense } from 'velox'; + +const fastData = await db.pageConfig.find(); + +// Slow query wrapped in Suspense โ€” the page shell renders immediately +const slowPostsPromise = db.posts.findMany({ include: { comments: true } }); +--- + +<main> + <h1>{fastData.title}</h1> + <Suspense fallback={<Spinner />}> + <PostList postsPromise={slowPostsPromise} /> + </Suspense> +</main> +``` + +The `<Suspense>` boundary renders its fallback immediately while the async data resolves, then streams the final HTML to the client. + +## Fetch Utilities + +Velox wraps the global `fetch` with several quality-of-life additions in server contexts: + +```typescript +import { fetch } from 'velox/server'; + +// Automatic base URL resolution for relative paths +const data = await fetch('/api/internal-endpoint'); + +// Request deduplication โ€” concurrent identical fetches share one in-flight request +const [a, b] = await Promise.all([fetch('/api/same'), fetch('/api/same')]); +// ^ only one HTTP request is made +``` diff --git a/sample-sites/velox-docs/pages/deploy-cloudflare.md b/sample-sites/velox-docs/pages/deploy-cloudflare.md new file mode 100644 index 0000000..3241035 --- /dev/null +++ b/sample-sites/velox-docs/pages/deploy-cloudflare.md @@ -0,0 +1,213 @@ +--- +title: Deploy to Cloudflare +sort: 110 +section-id: deployment +keywords: Cloudflare, Pages, Workers, deployment, edge, CDN, Wrangler +description: Deploying a Velox application to Cloudflare Pages and Workers +language: en +--- + +# Deploy to Cloudflare + +Velox runs natively on Cloudflare Workers. By combining Cloudflare Pages for static assets and Cloudflare Workers for server-side rendering, you get a globally distributed application with sub-millisecond cold starts. + +## Architecture + +Velox on Cloudflare uses: +- **Cloudflare Pages** โ€” serves static assets from the CDN edge +- **Cloudflare Workers** โ€” handles SSR requests at the edge +- **KV** โ€” used for ISR page caching +- **D1** โ€” SQLite-at-the-edge database (optional) +- **R2** โ€” object storage for user uploads (optional) + +## Setup + +Install the Cloudflare adapter: + +```bash +npm install @velox/cloudflare +npm install -D wrangler +``` + +Configure the adapter: + +```typescript +// velox.config.ts +import { defineConfig } from 'velox'; +import { cloudflare } from '@velox/cloudflare'; + +export default defineConfig({ + adapter: cloudflare({ + kvNamespace: 'VELOX_CACHE', // KV namespace for ISR cache + routes: { + exclude: ['/admin/*'], // serve these from Pages CDN only + }, + }), + build: { + target: 'edge', + }, +}); +``` + +## Wrangler Configuration + +Create `wrangler.toml`: + +```toml +name = "my-velox-app" +compatibility_date = "2026-01-01" +compatibility_flags = ["nodejs_compat"] +pages_build_output_dir = ".velox/output" + +[[kv_namespaces]] +binding = "VELOX_CACHE" +id = "your-kv-namespace-id" + +[[d1_databases]] +binding = "DB" +database_name = "my-database" +database_id = "your-database-id" + +[vars] +NODE_ENV = "production" +``` + +## Deployment Steps + +### Option 1: Cloudflare Pages Dashboard + +1. Go to [dash.cloudflare.com](https://dash.cloudflare.com) โ†’ **Pages** โ†’ **Create a project** +2. Connect your Git provider and select your repository +3. Set build settings: + - **Framework preset:** Velox + - **Build command:** `npm run build` + - **Build output directory:** `.velox/output` +4. Add environment variables +5. Click **Save and Deploy** + +### Option 2: Wrangler CLI + +```bash +# Log in +npx wrangler login + +# Build +npm run build + +# Deploy +npx wrangler pages deploy .velox/output --project-name my-velox-app + +# Or deploy as a Worker +npx wrangler deploy +``` + +## Using Cloudflare D1 (SQLite at the Edge) + +Cloudflare D1 is a serverless SQLite database that runs at the edge alongside your Workers. + +### Create a Database + +```bash +npx wrangler d1 create my-database +npx wrangler d1 execute my-database --file=./migrations/001_init.sql +``` + +### Query D1 in Velox + +```typescript +// routes/api/posts+server.ts +import { defineHandler } from 'velox/server'; + +export const GET = defineHandler(async (req) => { + // env.DB is automatically injected by the Cloudflare adapter + const { DB } = process.env as any; + const { results } = await DB.prepare('SELECT * FROM posts WHERE published = 1').all(); + return Response.json(results); +}); +``` + +Or with Drizzle's D1 driver: + +```typescript +import { drizzle } from 'drizzle-orm/d1'; +import * as schema from '$lib/schema'; + +export function getDB(env: Env) { + return drizzle(env.DB, { schema }); +} +``` + +## Using Cloudflare KV + +For key-value storage (feature flags, cached responses): + +```typescript +const value = await env.MY_KV.get('my-key'); +await env.MY_KV.put('my-key', JSON.stringify(data), { expirationTtl: 3600 }); +await env.MY_KV.delete('my-key'); +``` + +## Using Cloudflare R2 + +For file uploads and object storage: + +```typescript +// routes/api/upload+server.ts +import { defineHandler } from 'velox/server'; + +export const POST = defineHandler(async (req) => { + const formData = await req.formData(); + const file = formData.get('file') as File; + + const key = `uploads/${Date.now()}-${file.name}`; + await env.R2_BUCKET.put(key, file.stream(), { + httpMetadata: { contentType: file.type }, + }); + + return Response.json({ url: `https://assets.example.com/${key}` }); +}); +``` + +## Custom Domains + +1. Add your domain in Cloudflare's DNS settings (it should already be proxied through Cloudflare) +2. Go to Pages โ†’ Your project โ†’ **Custom domains** +3. Click **Set up a custom domain** and enter your domain +4. Cloudflare provisions SSL automatically + +For Workers, add a route in `wrangler.toml`: + +```toml +[[routes]] +pattern = "example.com/*" +zone_name = "example.com" +``` + +## Environment Variables + +Set secrets securely: + +```bash +npx wrangler secret put DATABASE_URL +npx wrangler secret put SESSION_SECRET +npx wrangler secret put JWT_SECRET +``` + +Non-secret variables go in `wrangler.toml` under `[vars]` or in the Pages dashboard. + +## Performance Tips + +- Use **Cache Rules** in the Cloudflare dashboard to cache SSR pages at the CDN layer +- Enable **Argo Smart Routing** for improved global routing +- Use **Tiered Cache** to reduce origin requests +- Set `Cache-Control: public, max-age=0, s-maxage=3600` on ISR pages to let Cloudflare cache them + +## Limits + +| Feature | Limit | +|---------|-------| +| Worker request timeout | 30s (CPU time) | +| Worker memory | 128 MB | +| KV value size | 25 MB | +| D1 database size | 2 GB | +| R2 object size | 5 GB | diff --git a/sample-sites/velox-docs/pages/deploy-docker.md b/sample-sites/velox-docs/pages/deploy-docker.md new file mode 100644 index 0000000..f189529 --- /dev/null +++ b/sample-sites/velox-docs/pages/deploy-docker.md @@ -0,0 +1,301 @@ +--- +title: Docker +sort: 120 +section-id: deployment +keywords: Docker, Dockerfile, docker-compose, container, containerisation, deployment +description: Containerising a Velox application with Docker and docker-compose +language: en +--- + +# Docker + +Containerising your Velox application with Docker makes it portable across any infrastructure โ€” from a simple VPS to a Kubernetes cluster. This guide covers building optimised Docker images and running multi-service stacks with docker-compose. + +## Dockerfile + +A production-grade multi-stage Dockerfile: + +```dockerfile +# Stage 1: Install dependencies +FROM node:22-alpine AS deps +WORKDIR /app + +# Copy package manifests only โ€” enables Docker layer caching +COPY package.json package-lock.json ./ +RUN npm ci --frozen-lockfile + +# Stage 2: Build the application +FROM node:22-alpine AS builder +WORKDIR /app + +COPY --from=deps /app/node_modules ./node_modules +COPY . . + +# Build args โ€” pass at build time +ARG PUBLIC_BASE_URL +ARG NODE_ENV=production +ENV PUBLIC_BASE_URL=$PUBLIC_BASE_URL +ENV NODE_ENV=$NODE_ENV + +RUN npm run build + +# Stage 3: Production runtime +FROM node:22-alpine AS runner +WORKDIR /app + +ENV NODE_ENV=production +ENV PORT=3700 + +# Create non-root user for security +RUN addgroup --system --gid 1001 veloxgroup && \ + adduser --system --uid 1001 veloxuser + +# Copy only production artefacts +COPY --from=builder --chown=veloxuser:veloxgroup /app/.velox/output ./ +COPY --from=builder --chown=veloxuser:veloxgroup /app/package.json ./ + +# Install production dependencies only +RUN npm ci --omit=dev --frozen-lockfile + +USER veloxuser + +EXPOSE 3700 + +CMD ["node", "server.js"] +``` + +## Building and Running + +```bash +# Build the image +docker build \ + --build-arg PUBLIC_BASE_URL=https://example.com \ + -t my-velox-app:latest . + +# Run the container +docker run \ + --env-file .env.production \ + -p 3700:3700 \ + --name velox-app \ + my-velox-app:latest + +# Run in background +docker run -d \ + --env-file .env.production \ + -p 3700:3700 \ + --restart unless-stopped \ + --name velox-app \ + my-velox-app:latest +``` + +## `.dockerignore` + +Exclude files from the build context to speed up builds: + +``` +node_modules +.velox +.git +.gitignore +*.md +.env +.env.* +!.env.example +tests +*.test.* +*.spec.* +coverage +``` + +## docker-compose + +A complete stack with the app, PostgreSQL, and Redis: + +```yaml +# docker-compose.yml +version: '3.9' + +services: + app: + build: + context: . + args: + PUBLIC_BASE_URL: ${PUBLIC_BASE_URL:-http://localhost:3700} + image: my-velox-app:latest + ports: + - "3700:3700" + environment: + NODE_ENV: production + DATABASE_URL: postgresql://velox:${DB_PASSWORD}@db:5432/velox + REDIS_URL: redis://redis:6379 + SESSION_SECRET: ${SESSION_SECRET} + JWT_SECRET: ${JWT_SECRET} + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:3700/api/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + + db: + image: postgres:16-alpine + environment: + POSTGRES_USER: velox + POSTGRES_PASSWORD: ${DB_PASSWORD} + POSTGRES_DB: velox + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U velox"] + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped + + redis: + image: redis:7-alpine + command: redis-server --requirepass ${REDIS_PASSWORD} --appendonly yes + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "--no-auth-warning", "-a", "${REDIS_PASSWORD}", "ping"] + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped + + nginx: + image: nginx:alpine + ports: + - "80:80" + - "443:443" + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ./nginx/ssl:/etc/nginx/ssl:ro + - nginx_cache:/var/cache/nginx + depends_on: + - app + restart: unless-stopped + +volumes: + postgres_data: + redis_data: + nginx_cache: +``` + +## Nginx Reverse Proxy + +```nginx +# nginx/nginx.conf +events { worker_connections 1024; } + +http { + upstream velox_app { + server app:3700; + keepalive 64; + } + + # Redirect HTTP โ†’ HTTPS + server { + listen 80; + server_name example.com www.example.com; + return 301 https://$host$request_uri; + } + + server { + listen 443 ssl http2; + server_name example.com www.example.com; + + ssl_certificate /etc/nginx/ssl/fullchain.pem; + ssl_certificate_key /etc/nginx/ssl/privkey.pem; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers HIGH:!aNULL:!MD5; + + # Static assets with long cache + location /assets/ { + proxy_pass http://velox_app; + add_header Cache-Control "public, max-age=31536000, immutable"; + } + + # Application + location / { + proxy_pass http://velox_app; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_cache_bypass $http_upgrade; + } + } +} +``` + +## Health Check Endpoint + +Add a health check to your application for Docker and load balancers: + +```typescript +// routes/api/health+server.ts +import { defineHandler, json } from 'velox/server'; +import { db } from '$lib/db'; + +export const GET = defineHandler(async () => { + try { + await db.$queryRaw`SELECT 1`; + return json({ status: 'ok', timestamp: new Date().toISOString() }); + } catch (error) { + return json({ status: 'error', error: String(error) }, { status: 503 }); + } +}); +``` + +## Running Database Migrations + +Run migrations as a separate one-shot container before starting the app: + +```yaml +# docker-compose.yml โ€” add this service + migrate: + image: my-velox-app:latest + command: npx prisma migrate deploy + environment: + DATABASE_URL: postgresql://velox:${DB_PASSWORD}@db:5432/velox + depends_on: + db: + condition: service_healthy + restart: "no" +``` + +## Container Best Practices + +| Practice | Why | +|----------|-----| +| Multi-stage builds | Reduces final image size by 60โ€“80% | +| Non-root user | Limits damage if the container is compromised | +| Read-only filesystem | Mount only what needs to be writable | +| `--restart unless-stopped` | Survives host reboots | +| Resource limits | Prevents a runaway container from affecting neighbours | +| Health checks | Enables zero-downtime rolling updates | + +Set resource limits: + +```yaml +app: + deploy: + resources: + limits: + cpus: '1.0' + memory: 512M + reservations: + cpus: '0.25' + memory: 128M +``` diff --git a/sample-sites/velox-docs/pages/deploy-self-hosted.md b/sample-sites/velox-docs/pages/deploy-self-hosted.md new file mode 100644 index 0000000..124a918 --- /dev/null +++ b/sample-sites/velox-docs/pages/deploy-self-hosted.md @@ -0,0 +1,275 @@ +--- +title: Self-Hosted +sort: 130 +section-id: deployment +keywords: self-hosted, VPS, nginx, PM2, systemd, Linux, server deployment +description: Running Velox on a VPS with nginx as a reverse proxy, managed by PM2 or systemd +language: en +--- + +# Self-Hosted + +Deploying Velox to your own server (a VPS, dedicated server, or on-premises machine) gives you full control over your infrastructure. This guide covers setting up a Linux server with nginx as a reverse proxy, and managing the Node.js process with either PM2 or systemd. + +## Server Preparation + +This guide assumes Ubuntu 24.04 LTS. Adjust package manager commands for other distributions. + +```bash +# Update the system +sudo apt update && sudo apt upgrade -y + +# Install Node.js (LTS) via nvm +curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.0/install.sh | bash +source ~/.bashrc +nvm install 22 +nvm use 22 +nvm alias default 22 + +# Install nginx +sudo apt install -y nginx + +# Install PostgreSQL (if needed) +sudo apt install -y postgresql postgresql-contrib + +# Install Redis (if needed) +sudo apt install -y redis-server +``` + +## Deploying the Application + +### Option A: Direct File Copy + +```bash +# On your local machine โ€” build the app +npm run build + +# Copy build output to the server +rsync -avz --delete .velox/output/ user@your-server.com:/var/www/my-velox-app/ +rsync -avz package.json package-lock.json user@your-server.com:/var/www/my-velox-app/ + +# On the server โ€” install production dependencies +cd /var/www/my-velox-app +npm ci --omit=dev +``` + +### Option B: Git + CI/CD + +Create a deploy script on the server: + +```bash +#!/bin/bash +# /home/deploy/deploy.sh +set -e + +APP_DIR=/var/www/my-velox-app +REPO=https://github.com/your-org/your-repo.git + +cd $APP_DIR + +# Pull latest code +git fetch --all +git reset --hard origin/main + +# Install dependencies +npm ci --frozen-lockfile --omit=dev + +# Build +npm run build + +# Restart application +pm2 reload my-velox-app --update-env + +echo "Deployment complete." +``` + +Call this from your GitHub Actions workflow: + +```yaml +# .github/workflows/deploy.yml +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Deploy to server + uses: appleboy/ssh-action@v1 + with: + host: ${{ secrets.SERVER_HOST }} + username: deploy + key: ${{ secrets.SERVER_SSH_KEY }} + script: /home/deploy/deploy.sh +``` + +## Environment Variables + +Store production secrets in `/etc/environment` or a `.env.production` file: + +```bash +# /etc/environment (system-wide) +NODE_ENV=production +DATABASE_URL=postgresql://velox:secretpass@localhost:5432/velox +SESSION_SECRET=your-long-random-secret-here +JWT_SECRET=another-long-random-secret + +# Or in a project-specific .env.production +sudo nano /var/www/my-velox-app/.env.production +``` + +Load the env file in your start command (covered below). + +## Process Management with PM2 + +PM2 is a battle-tested process manager for Node.js applications. + +```bash +npm install -g pm2 +``` + +Create a PM2 ecosystem file: + +```javascript +// /var/www/my-velox-app/ecosystem.config.js +module.exports = { + apps: [{ + name: 'my-velox-app', + script: 'server.js', + cwd: '/var/www/my-velox-app', + instances: 'max', // one process per CPU core + exec_mode: 'cluster', // enable cluster mode for zero-downtime restarts + env_file: '/var/www/my-velox-app/.env.production', + env: { + NODE_ENV: 'production', + PORT: 3700, + }, + error_file: '/var/log/pm2/my-velox-app-error.log', + out_file: '/var/log/pm2/my-velox-app-out.log', + log_date_format: 'YYYY-MM-DD HH:mm:ss', + max_memory_restart: '500M', // restart if memory exceeds 500 MB + }], +}; +``` + +Start and persist: + +```bash +pm2 start ecosystem.config.js +pm2 save # persist process list across reboots +pm2 startup # install systemd startup script + +# Useful commands +pm2 status +pm2 logs my-velox-app +pm2 reload my-velox-app # zero-downtime reload +pm2 restart my-velox-app # full restart +pm2 monit # real-time monitoring +``` + +## systemd Service (Alternative to PM2) + +For simpler setups, a systemd unit file: + +```ini +# /etc/systemd/system/my-velox-app.service +[Unit] +Description=My Velox Application +After=network.target postgresql.service redis.service + +[Service] +Type=simple +User=www-data +Group=www-data +WorkingDirectory=/var/www/my-velox-app +EnvironmentFile=/var/www/my-velox-app/.env.production +ExecStart=/usr/bin/node server.js +Restart=always +RestartSec=5 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=my-velox-app + +# Resource limits +LimitNOFILE=65536 +MemoryMax=512M + +[Install] +WantedBy=multi-user.target +``` + +```bash +sudo systemctl daemon-reload +sudo systemctl enable my-velox-app +sudo systemctl start my-velox-app +sudo journalctl -u my-velox-app -f # follow logs +``` + +## Nginx Configuration + +```nginx +# /etc/nginx/sites-available/my-velox-app +upstream velox { + server 127.0.0.1:3700; + keepalive 32; +} + +server { + listen 80; + server_name example.com www.example.com; + return 301 https://$host$request_uri; +} + +server { + listen 443 ssl http2; + server_name example.com www.example.com; + + ssl_certificate /etc/letsencrypt/live/example.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/example.com/privkey.pem; + ssl_session_cache shared:SSL:10m; + ssl_session_timeout 1d; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_prefer_server_ciphers off; + + gzip on; + gzip_types text/plain application/json application/javascript text/css; + + # Long-lived cache for hashed static assets + location ~ ^/assets/.*\.[0-9a-f]{8}\. { + proxy_pass http://velox; + add_header Cache-Control "public, max-age=31536000, immutable"; + } + + location / { + proxy_pass http://velox; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 60s; + proxy_send_timeout 60s; + } +} +``` + +```bash +sudo ln -s /etc/nginx/sites-available/my-velox-app /etc/nginx/sites-enabled/ +sudo nginx -t +sudo systemctl reload nginx +``` + +## SSL with Let's Encrypt + +```bash +sudo apt install -y certbot python3-certbot-nginx +sudo certbot --nginx -d example.com -d www.example.com +# Certbot automatically renews certificates via a cron job +``` + +## Firewall + +```bash +sudo ufw allow OpenSSH +sudo ufw allow 'Nginx Full' +sudo ufw enable +sudo ufw status +``` diff --git a/sample-sites/velox-docs/pages/deploy-vercel.md b/sample-sites/velox-docs/pages/deploy-vercel.md new file mode 100644 index 0000000..4823677 --- /dev/null +++ b/sample-sites/velox-docs/pages/deploy-vercel.md @@ -0,0 +1,183 @@ +--- +title: Deploy to Vercel +sort: 100 +section-id: deployment +keywords: Vercel, deployment, serverless, edge, CI/CD, deploy +description: Step-by-step guide to deploying a Velox application to Vercel +language: en +--- + +# Deploy to Vercel + +Vercel is the recommended hosting platform for Velox applications. The Velox Vercel adapter handles all configuration automatically โ€” no manual `vercel.json` setup is required for most projects. + +## Prerequisites + +- A Vercel account ([vercel.com](https://vercel.com)) +- The Vercel CLI: `npm install -g vercel` +- A Velox project committed to a Git repository (GitHub, GitLab, or Bitbucket) + +## Automatic Deployment via Git Integration + +### Step 1 โ€” Push Your Project to Git + +```bash +git init +git add . +git commit -m "Initial commit" +git remote add origin https://github.com/your-username/your-project.git +git push -u origin main +``` + +### Step 2 โ€” Import the Project on Vercel + +1. Go to [vercel.com/new](https://vercel.com/new) +2. Click **Add New Project** โ†’ **Import Git Repository** +3. Select your repository +4. Vercel automatically detects Velox and sets the correct build settings: + - **Framework Preset:** Velox + - **Build Command:** `velox build` + - **Output Directory:** `.velox/output` + - **Install Command:** `npm ci` + +### Step 3 โ€” Configure Environment Variables + +In the Vercel project dashboard: +1. Go to **Settings** โ†’ **Environment Variables** +2. Add each variable from your `.env.production` file + +Do **not** add `PUBLIC_BASE_URL` manually โ€” Vercel sets `VERCEL_URL` automatically and the Velox Vercel adapter uses it. + +### Step 4 โ€” Deploy + +Click **Deploy**. Vercel will: +1. Clone your repository +2. Install dependencies +3. Run `velox build` +4. Deploy to its global edge network + +Your site is live at `https://your-project.vercel.app`. + +## Vercel CLI Deployment + +For manual or scripted deployments: + +```bash +# Install adapter +npm install @velox/vercel + +# Deploy to preview (equivalent to a branch deploy) +vercel + +# Deploy to production +vercel --prod +``` + +## Configuring the Vercel Adapter + +Install and configure `@velox/vercel`: + +```bash +npm install @velox/vercel +``` + +```typescript +// velox.config.ts +import { defineConfig } from 'velox'; +import { vercel } from '@velox/vercel'; + +export default defineConfig({ + adapter: vercel({ + // Route-level edge function configuration + edgeRoutes: ['/api/stream', '/api/realtime'], + + // Override ISR revalidation for specific routes + isr: { + expiration: 60, // default revalidation (seconds) + bypassToken: process.env.VERCEL_ISR_BYPASS_TOKEN, + }, + + // Enable Vercel Image Optimisation (uses Vercel's CDN) + images: { + sizes: [640, 1080, 1920], + }, + }), +}); +``` + +## Edge Functions + +Mark individual routes to run on Vercel Edge instead of Node.js serverless: + +```typescript +// routes/api/stream+server.ts +export const config = { edge: true }; + +export const GET = defineHandler(async (req) => { + // Runs on Vercel Edge โ€” uses Web APIs only + return new Response('Hello from edge!'); +}); +``` + +Edge functions are deployed globally in ~70 Vercel regions and have ~0ms cold start. Use them for latency-sensitive, stateless handlers. + +## Custom Domains + +1. Go to your project on Vercel โ†’ **Settings** โ†’ **Domains** +2. Click **Add Domain** +3. Enter your domain (e.g., `example.com`) +4. Follow the DNS configuration instructions: + - Add a **CNAME** record: `www` โ†’ `cname.vercel-dns.com` + - Add an **A** record: `@` โ†’ `76.76.21.21` +5. SSL certificates are provisioned automatically via Let's Encrypt + +Set the canonical base URL environment variable in Vercel: + +``` +PUBLIC_BASE_URL = https://example.com +``` + +## Preview Deployments + +Every pull request gets an automatic preview deployment at a unique URL (`https://your-project-git-branch-name.vercel.app`). This is configured by default and requires no additional setup. + +To share preview deployments with your team, enable **Password Protection** under **Settings** โ†’ **Deployment Protection**. + +## Build Cache + +Velox's Velocitor build cache is automatically persisted across deployments by the Vercel adapter. Subsequent deployments typically build 3โ€“5ร— faster than the first. + +## `vercel.json` Reference + +For advanced configuration, create a `vercel.json` at your project root: + +```json +{ + "regions": ["iad1", "fra1"], + "cleanUrls": true, + "trailingSlash": false, + "headers": [ + { + "source": "/assets/(.*)", + "headers": [ + { "key": "Cache-Control", "value": "public, max-age=31536000, immutable" } + ] + } + ], + "redirects": [ + { "source": "/old-path", "destination": "/new-path", "permanent": true } + ] +} +``` + +## Monitoring and Analytics + +Enable Vercel Analytics and Speed Insights in your project dashboard. The Velox adapter hooks into these automatically โ€” no code changes needed. + +For custom event tracking: + +```typescript +import { track } from '@vercel/analytics'; + +track('button_clicked', { component: 'Hero', variant: 'primary' }); +``` diff --git a/sample-sites/velox-docs/pages/guide-auth.md b/sample-sites/velox-docs/pages/guide-auth.md new file mode 100644 index 0000000..fa10365 --- /dev/null +++ b/sample-sites/velox-docs/pages/guide-auth.md @@ -0,0 +1,288 @@ +--- +title: Authentication +sort: 100 +section-id: guides +keywords: authentication, JWT, OAuth2, session, login, auth, security +description: Implementing authentication in Velox using JWT, OAuth2, and session-based strategies +language: en +--- + +# Authentication + +This guide covers the three most common authentication patterns for Velox applications: session-based auth, JWT-based auth, and OAuth2 with third-party providers. + +## Session-Based Authentication + +Session auth stores user state server-side. The client holds only a signed session cookie. This is the simplest and most secure approach for most web applications. + +### Setup + +Install the session plugin: + +```bash +npm install @velox/session @velox/session-redis +``` + +Configure in `velox.config.ts`: + +```typescript +import { RedisSessionStore } from '@velox/session-redis'; + +export default defineConfig({ + session: { + secret: process.env.SESSION_SECRET!, + cookieName: 'app.session', + maxAge: 60 * 60 * 24 * 7, // 7 days + httpOnly: true, + secure: process.env.NODE_ENV === 'production', + sameSite: 'lax', + store: new RedisSessionStore({ url: process.env.REDIS_URL! }), + }, +}); +``` + +### Login Handler + +```typescript +// routes/api/auth/login+server.ts +import { defineHandler, json, redirect } from 'velox/server'; +import { useSession } from 'velox/server'; +import { db } from '$lib/db'; +import { verifyPassword } from '$lib/crypto'; + +export const POST = defineHandler(async (req) => { + const { email, password } = await req.json(); + + if (!email || !password) { + return json({ error: 'Email and password are required' }, { status: 400 }); + } + + const user = await db.users.findByEmail(email); + if (!user || !(await verifyPassword(password, user.passwordHash))) { + return json({ error: 'Invalid credentials' }, { status: 401 }); + } + + const session = await useSession(); + await session.regenerate(); // prevent session fixation + await session.update({ userId: user.id, role: user.role }); + + return json({ ok: true, redirectTo: '/dashboard' }); +}); +``` + +### Auth Middleware + +```typescript +// middleware/auth.ts +import { defineMiddleware, redirect } from 'velox/server'; +import { useSession } from 'velox/server'; + +const PUBLIC_PATHS = ['/login', '/register', '/api/auth']; + +export default defineMiddleware(async ({ request, next }) => { + const pathname = new URL(request.url).pathname; + + if (PUBLIC_PATHS.some(p => pathname.startsWith(p))) { + return next(); + } + + const session = await useSession<{ userId: string }>(); + + if (!session.data.userId) { + return redirect(`/login?next=${encodeURIComponent(pathname)}`); + } + + request.context.set('userId', session.data.userId); + return next(); +}); +``` + +## JWT Authentication + +JWT is stateless and ideal for API-first applications or mobile/SPA backends. + +### Generating Tokens + +```typescript +// lib/jwt.ts +import { SignJWT, jwtVerify } from 'jose'; + +const secret = new TextEncoder().encode(process.env.JWT_SECRET!); + +export interface JWTPayload { + sub: string; // user ID + role: string; + iat: number; + exp: number; +} + +export async function signToken(userId: string, role: string): Promise<string> { + return new SignJWT({ sub: userId, role }) + .setProtectedHeader({ alg: 'HS256' }) + .setIssuedAt() + .setExpirationTime('15m') + .sign(secret); +} + +export async function signRefreshToken(userId: string): Promise<string> { + return new SignJWT({ sub: userId, type: 'refresh' }) + .setProtectedHeader({ alg: 'HS256' }) + .setIssuedAt() + .setExpirationTime('30d') + .sign(secret); +} + +export async function verifyToken(token: string): Promise<JWTPayload> { + const { payload } = await jwtVerify(token, secret); + return payload as JWTPayload; +} +``` + +### Login + Refresh Endpoints + +```typescript +// routes/api/auth/token+server.ts +import { defineHandler, json, badRequest, unauthorized } from 'velox/server'; +import { signToken, signRefreshToken, verifyToken } from '$lib/jwt'; +import { db } from '$lib/db'; +import { verifyPassword } from '$lib/crypto'; + +export const POST = defineHandler(async (req) => { + const { grant_type, email, password, refresh_token } = await req.json(); + + if (grant_type === 'password') { + const user = await db.users.findByEmail(email); + if (!user || !(await verifyPassword(password, user.passwordHash))) { + return unauthorized('Invalid credentials'); + } + return json({ + access_token: await signToken(user.id, user.role), + refresh_token: await signRefreshToken(user.id), + token_type: 'Bearer', + expires_in: 900, + }); + } + + if (grant_type === 'refresh_token') { + const payload = await verifyToken(refresh_token).catch(() => null); + if (!payload || payload.type !== 'refresh') { + return unauthorized('Invalid refresh token'); + } + const user = await db.users.findById(payload.sub); + return json({ + access_token: await signToken(user.id, user.role), + token_type: 'Bearer', + expires_in: 900, + }); + } + + return badRequest('Unsupported grant_type'); +}); +``` + +## OAuth2 with Third-Party Providers + +### Using `@velox/auth` + +```bash +npm install @velox/auth +``` + +Configure providers: + +```typescript +// lib/auth.ts +import { createAuth } from '@velox/auth'; + +export const auth = createAuth({ + providers: [ + { + id: 'github', + clientId: process.env.GITHUB_CLIENT_ID!, + clientSecret: process.env.GITHUB_CLIENT_SECRET!, + authorizationUrl: 'https://github.com/login/oauth/authorize', + tokenUrl: 'https://github.com/login/oauth/access_token', + userInfoUrl: 'https://api.github.com/user', + scopes: ['user:email'], + }, + { + id: 'google', + clientId: process.env.GOOGLE_CLIENT_ID!, + clientSecret: process.env.GOOGLE_CLIENT_SECRET!, + // Google uses OpenID Connect โ€” most fields are auto-discovered + }, + ], + callbacks: { + async onUserInfo(provider, userInfo) { + const user = await db.users.upsert({ + where: { email: userInfo.email }, + create: { email: userInfo.email, name: userInfo.name, provider }, + update: { name: userInfo.name }, + }); + return { id: user.id, role: user.role }; + }, + }, +}); +``` + +OAuth callback route: + +```typescript +// routes/auth/[provider]/callback+server.ts +import { defineHandler } from 'velox/server'; +import { auth } from '$lib/auth'; + +export const GET = defineHandler(async (req) => { + return auth.handleCallback(req); +}); +``` + +## Password Hashing + +Always use a slow hashing algorithm. Velox recommends Argon2: + +```typescript +// lib/crypto.ts +import { hash, verify } from '@node-rs/argon2'; + +export async function hashPassword(password: string): Promise<string> { + return hash(password, { + memoryCost: 19456, + timeCost: 2, + outputLen: 32, + parallelism: 1, + }); +} + +export async function verifyPassword(password: string, hash: string): Promise<boolean> { + return verify(hash, password); +} +``` + +## Role-Based Access Control + +```typescript +// middleware/rbac.ts +import { defineMiddleware, forbidden } from 'velox/server'; + +const ROUTE_ROLES: Record<string, string[]> = { + '/admin': ['admin'], + '/api/admin': ['admin'], + '/api/reports': ['admin', 'analyst'], +}; + +export default defineMiddleware(async ({ request, next }) => { + const pathname = new URL(request.url).pathname; + const requiredRoles = Object.entries(ROUTE_ROLES) + .find(([path]) => pathname.startsWith(path))?.[1]; + + if (!requiredRoles) return next(); + + const user = request.context.get('user') as { role: string } | undefined; + if (!user || !requiredRoles.includes(user.role)) { + return forbidden('Insufficient permissions'); + } + + return next(); +}); +``` diff --git a/sample-sites/velox-docs/pages/guide-database.md b/sample-sites/velox-docs/pages/guide-database.md new file mode 100644 index 0000000..6f7d9c7 --- /dev/null +++ b/sample-sites/velox-docs/pages/guide-database.md @@ -0,0 +1,285 @@ +--- +title: Database Integration +sort: 110 +section-id: guides +keywords: database, Prisma, DrizzleORM, SQL, ORM, PostgreSQL, database integration +description: How to integrate databases in Velox using Prisma, DrizzleORM, or raw SQL +language: en +--- + +# Database Integration + +Velox is database-agnostic. This guide covers the three most popular approaches: Prisma (full-featured ORM), DrizzleORM (lightweight TypeScript-first ORM), and raw SQL with a typed query builder. + +## Prisma + +Prisma is the most popular ORM in the Node.js ecosystem. It provides a schema-first approach, auto-generated type-safe client, and powerful migrations. + +### Setup + +```bash +npm install prisma @prisma/client +npx prisma init +``` + +This creates a `prisma/schema.prisma` file. Example schema: + +```prisma +// prisma/schema.prisma +generator client { + provider = "prisma-client-js" +} + +datasource db { + provider = "postgresql" + url = env("DATABASE_URL") +} + +model User { + id String @id @default(cuid()) + email String @unique + name String? + role Role @default(USER) + posts Post[] + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt +} + +model Post { + id String @id @default(cuid()) + title String + slug String @unique + content String + published Boolean @default(false) + author User @relation(fields: [authorId], references: [id]) + authorId String + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt +} + +enum Role { + USER + ADMIN + ANALYST +} +``` + +Generate and run the initial migration: + +```bash +npx prisma migrate dev --name init +npx prisma generate +``` + +### Database Client Singleton + +In a server environment, always reuse a single Prisma client instance: + +```typescript +// lib/db.ts +import { PrismaClient } from '@prisma/client'; + +const globalForPrisma = globalThis as unknown as { prisma: PrismaClient }; + +export const db = globalForPrisma.prisma ?? new PrismaClient({ + log: process.env.NODE_ENV === 'development' ? ['query', 'warn', 'error'] : ['error'], +}); + +if (process.env.NODE_ENV !== 'production') globalForPrisma.prisma = db; +``` + +### Usage in Routes + +```typescript +// routes/blog/[slug].velox server block +import { db } from '$lib/db'; + +const post = await db.post.findUnique({ + where: { slug: params.slug, published: true }, + include: { author: { select: { name: true } } }, +}); + +if (!post) throw notFound(); +``` + +### Transactions + +```typescript +const result = await db.$transaction(async (tx) => { + const user = await tx.user.create({ data: { email, name } }); + const profile = await tx.profile.create({ data: { userId: user.id } }); + return { user, profile }; +}); +``` + +## DrizzleORM + +DrizzleORM is a TypeScript-first ORM with a SQL-like query API and zero overhead. + +### Setup + +```bash +npm install drizzle-orm postgres +npm install -D drizzle-kit +``` + +Define your schema in TypeScript: + +```typescript +// lib/schema.ts +import { pgTable, text, boolean, timestamp, pgEnum } from 'drizzle-orm/pg-core'; + +export const roleEnum = pgEnum('role', ['user', 'admin', 'analyst']); + +export const users = pgTable('users', { + id: text('id').primaryKey().$defaultFn(() => crypto.randomUUID()), + email: text('email').notNull().unique(), + name: text('name'), + role: roleEnum('role').default('user').notNull(), + createdAt: timestamp('created_at').defaultNow().notNull(), + updatedAt: timestamp('updated_at').defaultNow().notNull(), +}); + +export const posts = pgTable('posts', { + id: text('id').primaryKey().$defaultFn(() => crypto.randomUUID()), + title: text('title').notNull(), + slug: text('slug').notNull().unique(), + content: text('content').notNull(), + published: boolean('published').default(false).notNull(), + authorId: text('author_id').notNull().references(() => users.id), + createdAt: timestamp('created_at').defaultNow().notNull(), +}); +``` + +### Database Client + +```typescript +// lib/db.ts +import { drizzle } from 'drizzle-orm/postgres-js'; +import postgres from 'postgres'; +import * as schema from './schema'; + +const client = postgres(process.env.DATABASE_URL!); +export const db = drizzle(client, { schema }); +``` + +### Querying + +```typescript +import { db } from '$lib/db'; +import { posts, users } from '$lib/schema'; +import { eq, and, desc } from 'drizzle-orm'; + +// Find one +const post = await db.query.posts.findFirst({ + where: and(eq(posts.slug, slug), eq(posts.published, true)), + with: { author: { columns: { name: true } } }, +}); + +// Find many with ordering +const recentPosts = await db + .select() + .from(posts) + .where(eq(posts.published, true)) + .orderBy(desc(posts.createdAt)) + .limit(10); + +// Insert +const [newPost] = await db + .insert(posts) + .values({ title, slug, content, authorId }) + .returning(); + +// Update +await db + .update(posts) + .set({ published: true, updatedAt: new Date() }) + .where(eq(posts.id, postId)); + +// Delete +await db.delete(posts).where(eq(posts.id, postId)); +``` + +### Migrations + +Configure Drizzle Kit: + +```typescript +// drizzle.config.ts +import { defineConfig } from 'drizzle-kit'; + +export default defineConfig({ + schema: './lib/schema.ts', + out: './drizzle', + dialect: 'postgresql', + dbCredentials: { url: process.env.DATABASE_URL! }, +}); +``` + +```bash +npx drizzle-kit generate +npx drizzle-kit migrate +``` + +## Raw SQL with `postgres.js` + +For maximum control and performance with complex queries: + +```typescript +// lib/db.ts +import postgres from 'postgres'; + +export const sql = postgres(process.env.DATABASE_URL!, { + max: 20, // connection pool size + idle_timeout: 20, // seconds before idle connection closes + connect_timeout: 10, // connection timeout + types: { + bigint: postgres.BigInt, // return BigInt instead of string + }, +}); +``` + +Usage: + +```typescript +import { sql } from '$lib/db'; + +// Parameterised query (safe from SQL injection) +const users = await sql<User[]>` + SELECT id, email, name, role + FROM users + WHERE role = ${role} + ORDER BY created_at DESC + LIMIT ${limit} +`; + +// Transaction +const result = await sql.begin(async (sql) => { + const [user] = await sql` + INSERT INTO users (email, name) + VALUES (${email}, ${name}) + RETURNING * + `; + await sql` + INSERT INTO audit_log (user_id, action) + VALUES (${user.id}, 'register') + `; + return user; +}); +``` + +## Connection Pooling in Production + +For serverless or edge deployments, use a connection pooler: + +- **PgBouncer** โ€” a lightweight PostgreSQL connection pooler for VPS deployments +- **Supabase Supavisor** โ€” serverless-aware pooler built for transactional workloads +- **Neon / PlanetScale** โ€” managed databases with built-in HTTP-based connection pooling + +```typescript +// For serverless (e.g. Vercel, Cloudflare) โ€” use HTTP-based driver +import { neon } from '@neondatabase/serverless'; + +const sql = neon(process.env.DATABASE_URL!); +const users = await sql`SELECT * FROM users LIMIT 10`; +``` diff --git a/sample-sites/velox-docs/pages/guide-i18n.md b/sample-sites/velox-docs/pages/guide-i18n.md new file mode 100644 index 0000000..b9572a8 --- /dev/null +++ b/sample-sites/velox-docs/pages/guide-i18n.md @@ -0,0 +1,261 @@ +--- +title: Internationalisation +sort: 130 +section-id: guides +keywords: i18n, internationalisation, localisation, translation, locale routing, multilingual +description: Setting up internationalisation in Velox โ€” translation files, locale routing, and pluralisation +language: en +--- + +# Internationalisation + +Velox has built-in internationalisation (i18n) support through `@velox/i18n`. It handles locale detection, URL-based locale routing, typed translation files, and pluralisation. + +## Setup + +```bash +npm install @velox/i18n +``` + +Configure in `velox.config.ts`: + +```typescript +import { defineConfig } from 'velox'; + +export default defineConfig({ + i18n: { + locales: ['en', 'fr', 'de', 'ja'], + defaultLocale: 'en', + routing: 'prefix-except-default', + // 'prefix': all locales get a prefix (/en, /fr, /de, /ja) + // 'prefix-except-default': default locale has no prefix + // 'domain': different domains per locale + messagesDir: 'messages', + }, +}); +``` + +## Translation Files + +Create a `messages/` directory at your project root: + +``` +messages/ +โ”œโ”€โ”€ en.json +โ”œโ”€โ”€ fr.json +โ”œโ”€โ”€ de.json +โ””โ”€โ”€ ja.json +``` + +Translation files use a flat or nested key structure: + +```json +// messages/en.json +{ + "nav.home": "Home", + "nav.blog": "Blog", + "nav.about": "About", + "home.hero.title": "Build faster with Velox", + "home.hero.subtitle": "The TypeScript framework for the modern web", + "home.cta": "Get started", + "post.readMore": "Read more", + "post.publishedOn": "Published on {date}", + "post.comments": "{count, plural, =0{No comments} =1{1 comment} other{# comments}}", + "auth.loginButton": "Log in", + "auth.logoutButton": "Log out", + "errors.notFound": "Page not found", + "errors.serverError": "Something went wrong" +} +``` + +```json +// messages/fr.json +{ + "nav.home": "Accueil", + "nav.blog": "Blog", + "nav.about": "ร€ propos", + "home.hero.title": "Construisez plus vite avec Velox", + "home.hero.subtitle": "Le framework TypeScript pour le web moderne", + "home.cta": "Commencer", + "post.readMore": "Lire la suite", + "post.publishedOn": "Publiรฉ le {date}", + "post.comments": "{count, plural, =0{Aucun commentaire} =1{1 commentaire} other{# commentaires}}", + "auth.loginButton": "Se connecter", + "auth.logoutButton": "Se dรฉconnecter", + "errors.notFound": "Page introuvable", + "errors.serverError": "Une erreur est survenue" +} +``` + +## Using Translations + +### In Server Blocks + +```tsx +--- +import { useTranslations } from 'velox/i18n'; + +const t = useTranslations(); +const locale = useLocale().locale; +--- + +<main> + <h1>{t('home.hero.title')}</h1> + <p>{t('home.hero.subtitle')}</p> + <a href="/docs">{t('home.cta')}</a> +</main> +``` + +### With Parameters + +```tsx +--- +const t = useTranslations(); +const publishedDate = new Intl.DateTimeFormat(locale).format(post.createdAt); +--- + +<p>{t('post.publishedOn', { date: publishedDate })}</p> +``` + +### Pluralisation + +Velox uses the ICU message format for pluralisation: + +```typescript +t('post.comments', { count: 0 }); // "No comments" +t('post.comments', { count: 1 }); // "1 comment" +t('post.comments', { count: 42 }); // "42 comments" +``` + +### In Client Components + +```typescript +import { useTranslations } from 'velox/i18n/client'; + +export default function LikeButton({ postId }: { postId: string }) { + const t = useTranslations(); + const liked = signal(false); + + return ( + <button onClick={() => liked.value = !liked.value}> + {liked.value ? t('post.liked') : t('post.like')} + </button> + ); +} +``` + +## Locale Routing + +With `routing: 'prefix-except-default'` and `defaultLocale: 'en'`: + +| URL | Locale | +|-----|--------| +| `/` | `en` | +| `/about` | `en` | +| `/fr` | `fr` | +| `/fr/about` | `fr` | +| `/de/blog/my-post` | `de` | + +Velox automatically generates alternate hreflang links for SEO. + +## Locale Switcher Component + +```tsx +import { useLocale, usePathname } from 'velox/i18n/client'; + +export default function LocaleSwitcher() { + const { locale, locales } = useLocale(); + const pathname = usePathname(); + + return ( + <div class="locale-switcher"> + {locales.map(loc => ( + <a + key={loc} + href={getLocalizedPath(pathname.value, loc)} + class={loc === locale ? 'active' : ''} + hreflang={loc} + > + {getLocaleLabel(loc)} + </a> + ))} + </div> + ); +} + +function getLocaleLabel(locale: string): string { + const labels: Record<string, string> = { + en: 'English', + fr: 'Franรงais', + de: 'Deutsch', + ja: 'ๆ—ฅๆœฌ่ชž', + }; + return labels[locale] ?? locale; +} +``` + +## Domain-Based Routing + +For country-specific top-level domains: + +```typescript +export default defineConfig({ + i18n: { + locales: ['en', 'fr', 'de'], + defaultLocale: 'en', + routing: 'domain', + domains: { + en: 'example.com', + fr: 'example.fr', + de: 'example.de', + }, + }, +}); +``` + +## Type-Safe Translation Keys + +Generate a TypeScript type for your translation keys: + +```bash +npx velox i18n:generate-types +``` + +This creates `.velox/types/i18n.d.ts` so that `t('invalid.key')` is a compile-time error. + +## RTL Languages + +For right-to-left languages (Arabic, Hebrew, etc.), add the `dir` attribute dynamically: + +```tsx +// layouts/default.velox +--- +const { locale } = useLocale(); +const isRTL = ['ar', 'he', 'fa'].includes(locale); +--- + +<html lang={locale} dir={isRTL ? 'rtl' : 'ltr'}> + ... +</html> +``` + +## Date, Number, and Currency Formatting + +Use the `Intl` APIs with the current locale: + +```typescript +import { useLocale } from 'velox/i18n'; + +const { locale } = useLocale(); + +// Dates +const dateFormatter = new Intl.DateTimeFormat(locale, { dateStyle: 'long' }); +const formatted = dateFormatter.format(new Date(post.createdAt)); + +// Currency +const priceFormatter = new Intl.NumberFormat(locale, { + style: 'currency', + currency: 'USD', +}); +const price = priceFormatter.format(product.price); +``` diff --git a/sample-sites/velox-docs/pages/guide-performance.md b/sample-sites/velox-docs/pages/guide-performance.md new file mode 100644 index 0000000..55bc9b7 --- /dev/null +++ b/sample-sites/velox-docs/pages/guide-performance.md @@ -0,0 +1,283 @@ +--- +title: Performance +sort: 140 +section-id: guides +keywords: performance, code splitting, lazy loading, caching, optimisation, Core Web Vitals +description: Performance optimisation strategies for Velox apps โ€” code splitting, lazy loading, and caching +language: en +--- + +# Performance + +![Performance Dashboard](assets/images/performance.jpg) + +Velox is designed to be fast by default. The Rust-based compiler handles tree-shaking, code splitting, and asset optimisation automatically. This guide covers the additional strategies you can apply to push your application to peak performance. + +## Core Web Vitals Targets + +Before optimising, establish baselines. Aim for: + +| Metric | Good | Needs Work | +|--------|------|-----------| +| LCP (Largest Contentful Paint) | โ‰ค 2.5s | > 4.0s | +| FID / INP (Interaction to Next Paint) | โ‰ค 200ms | > 500ms | +| CLS (Cumulative Layout Shift) | โ‰ค 0.1 | > 0.25 | +| TTFB (Time to First Byte) | โ‰ค 800ms | > 1800ms | + +Use Velox's built-in analytics to monitor these in production: + +```typescript +// velox.config.ts +export default defineConfig({ + analytics: { + webVitals: true, + endpoint: '/api/analytics/vitals', + }, +}); +``` + +## Code Splitting + +Velox automatically splits your JavaScript bundle by route. Every route gets its own chunk, and shared code is extracted into a common chunk. You do not need to configure this. + +For further control, use dynamic imports: + +```typescript +// Only loads when the user clicks "Open" +const HeavyEditor = lazy(() => import('./HeavyEditor')); + +function PostPage() { + const isEditing = signal(false); + return ( + <div> + <h1>{post.title}</h1> + {isEditing.value && ( + <Suspense fallback={<Skeleton />}> + <HeavyEditor post={post} /> + </Suspense> + )} + <button onClick={() => isEditing.value = true}>Edit</button> + </div> + ); +} +``` + +## Lazy Hydration + +Delay hydration of interactive components until they are needed: + +```tsx +<!-- Hydrate only when the component scrolls into view --> +<CommentSection client:visible postId={post.id} /> + +<!-- Hydrate only when the browser is idle --> +<AnalyticsWidget client:idle /> + +<!-- Hydrate only when a CSS media query matches --> +<MobileNav client:media="(max-width: 768px)" /> + +<!-- Never hydrate โ€” purely server-rendered, no JS --> +<StaticSidebar /> +``` + +The less JavaScript you ship to the client, the better the INP score. + +## Image Optimisation + +Enable image optimisation in `velox.config.ts`: + +```typescript +assets: { + imageOptimisation: { + enabled: true, + formats: ['webp', 'avif'], + quality: 85, + maxWidth: 2000, + }, +}, +``` + +Use the `<Image>` component for automatic width/height and format negotiation: + +```tsx +import { Image } from 'velox'; + +<Image + src="/assets/images/hero.jpg" + alt="Hero image" + width={1200} + height={400} + priority // preload this image (use for above-the-fold images) + sizes="(max-width: 768px) 100vw, 1200px" +/> +``` + +The `priority` prop adds a `<link rel="preload">` tag to the document head and marks the image as `fetchpriority="high"`, which is the single biggest LCP improvement for image-heavy pages. + +## HTTP Caching + +Set appropriate `Cache-Control` headers for your routes: + +```typescript +// In a route server block +response.headers.set('Cache-Control', 'public, max-age=3600, stale-while-revalidate=86400'); + +// Or via config for specific paths +export default defineConfig({ + headers: [ + { + source: '/assets/*', + headers: [ + { key: 'Cache-Control', value: 'public, max-age=31536000, immutable' }, + ], + }, + { + source: '/', + headers: [ + { key: 'Cache-Control', value: 'public, max-age=0, s-maxage=3600, stale-while-revalidate=86400' }, + ], + }, + ], +}); +``` + +## Server-Side Caching + +Cache expensive computations and database queries: + +```typescript +import { useCache } from 'velox/server'; + +const cache = useCache(); + +async function getPopularPosts() { + const cached = await cache.get<Post[]>('posts:popular'); + if (cached) return cached; + + const posts = await db.posts.findMany({ + where: { published: true }, + orderBy: { viewCount: 'desc' }, + take: 10, + }); + + await cache.set('posts:popular', posts, { ttl: 300 }); // 5 minutes + return posts; +} +``` + +## Database Query Optimisation + +Common patterns that prevent N+1 queries: + +```typescript +// Bad: N+1 query +const posts = await db.post.findMany(); +const postsWithAuthors = await Promise.all( + posts.map(p => db.user.findById(p.authorId)) // N extra queries! +); + +// Good: single query with include +const postsWithAuthors = await db.post.findMany({ + include: { author: { select: { id: true, name: true } } }, +}); +``` + +Add indexes for commonly filtered columns: + +```sql +-- In a migration +CREATE INDEX CONCURRENTLY idx_posts_published_created + ON posts (published, created_at DESC) + WHERE published = true; +``` + +## Edge Deployment + +Deploy to edge locations close to your users: + +```typescript +// velox.config.ts +export default defineConfig({ + build: { + target: 'edge', + }, +}); +``` + +Edge-compatible routes must use Web APIs only: + +```typescript +// โœ… Edge-compatible +import { defineHandler } from 'velox/server'; +export const GET = defineHandler(async (req) => { + const data = await fetch('https://api.example.com/data'); + return Response.json(await data.json()); +}); + +// โŒ Not edge-compatible (uses Node.js built-ins) +import fs from 'node:fs'; +``` + +## Bundle Analysis + +Generate and review a bundle analysis report: + +```bash +ANALYZE=1 npm run build +# Opens .velox/output/analyze.html in your browser +``` + +Look for: +- Unexpectedly large dependencies (replace with smaller alternatives) +- Duplicate dependencies at different versions +- Client-side imports of server-only packages + +## Prefetching + +Configure link prefetching globally: + +```typescript +export default defineConfig({ + prefetch: { + defaultStrategy: 'hover', // 'hover' | 'viewport' | false + concurrency: 2, // max concurrent prefetch requests + ignore: ['/admin/*', '/api/*'], + }, +}); +``` + +## Font Loading + +Avoid layout shift from font loading: + +```css +/* styles/global.css */ +@font-face { + font-family: 'Inter'; + src: url('/fonts/inter-var.woff2') format('woff2'); + font-weight: 100 900; + font-display: swap; /* show fallback text immediately */ +} +``` + +```tsx +// layouts/default.velox +<Head> + <link rel="preload" href="/fonts/inter-var.woff2" as="font" type="font/woff2" crossOrigin="" /> +</Head> +``` + +## Performance Monitoring + +Integrate with monitoring services: + +```typescript +// lib/monitoring.ts +export function reportWebVitals({ name, value, id }: Metric) { + fetch('/api/analytics/vitals', { + method: 'POST', + body: JSON.stringify({ name, value, id, page: window.location.pathname }), + keepalive: true, + }); +} +``` diff --git a/sample-sites/velox-docs/pages/guide-testing.md b/sample-sites/velox-docs/pages/guide-testing.md new file mode 100644 index 0000000..2fa68a5 --- /dev/null +++ b/sample-sites/velox-docs/pages/guide-testing.md @@ -0,0 +1,327 @@ +--- +title: Testing +sort: 120 +section-id: guides +keywords: testing, unit tests, integration tests, E2E, Playwright, Vitest, testing strategy +description: Testing Velox applications with unit tests, integration tests, and end-to-end tests using Playwright +language: en +--- + +# Testing + +Velox integrates with Vitest for unit and integration tests, and Playwright for end-to-end tests. The `@velox/test` package provides additional test utilities tailored for Velox's server blocks and API routes. + +## Setup + +```bash +npm install -D vitest @velox/test @playwright/test +npx playwright install +``` + +Add test scripts to `package.json`: + +```json +{ + "scripts": { + "test": "vitest", + "test:ui": "vitest --ui", + "test:e2e": "playwright test", + "test:coverage": "vitest --coverage" + } +} +``` + +Configure Vitest: + +```typescript +// vitest.config.ts +import { defineConfig } from 'vitest/config'; +import { veloxTestPlugin } from '@velox/test'; + +export default defineConfig({ + plugins: [veloxTestPlugin()], + test: { + environment: 'node', + globals: true, + setupFiles: ['./tests/setup.ts'], + coverage: { + provider: 'v8', + reporter: ['text', 'lcov'], + }, + }, +}); +``` + +## Unit Tests + +### Testing Utility Functions + +```typescript +// lib/formatters.ts +export function formatCurrency(amount: number, currency = 'USD'): string { + return new Intl.NumberFormat('en-US', { style: 'currency', currency }).format(amount); +} +``` + +```typescript +// tests/unit/formatters.test.ts +import { describe, it, expect } from 'vitest'; +import { formatCurrency } from '$lib/formatters'; + +describe('formatCurrency', () => { + it('formats USD amounts', () => { + expect(formatCurrency(1000)).toBe('$1,000.00'); + expect(formatCurrency(9.99)).toBe('$9.99'); + }); + + it('formats other currencies', () => { + expect(formatCurrency(1000, 'EUR')).toBe('โ‚ฌ1,000.00'); + }); + + it('handles zero', () => { + expect(formatCurrency(0)).toBe('$0.00'); + }); +}); +``` + +### Testing Components + +```tsx +// tests/unit/Counter.test.tsx +import { describe, it, expect } from 'vitest'; +import { render, fireEvent } from '@velox/test'; +import Counter from '$components/Counter'; + +describe('Counter', () => { + it('renders with initial value', () => { + const { getByText } = render(<Counter initialValue={5} />); + expect(getByText('5')).toBeTruthy(); + }); + + it('increments on button click', async () => { + const { getByText, getByRole } = render(<Counter initialValue={0} />); + await fireEvent.click(getByRole('button', { name: /increment/i })); + expect(getByText('1')).toBeTruthy(); + }); + + it('decrements below initial value', async () => { + const { getByText, getByRole } = render(<Counter initialValue={3} />); + await fireEvent.click(getByRole('button', { name: /decrement/i })); + expect(getByText('2')).toBeTruthy(); + }); +}); +``` + +## Integration Tests + +### Testing API Routes + +The `createTestServer` utility starts a real Velox server for integration testing: + +```typescript +// tests/integration/api/users.test.ts +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { createTestServer } from '@velox/test'; +import { db } from '$lib/db'; + +let server: Awaited<ReturnType<typeof createTestServer>>; + +beforeAll(async () => { + server = await createTestServer({ seed: true }); +}); + +afterAll(async () => { + await server.stop(); + await db.$disconnect(); +}); + +describe('GET /api/users', () => { + it('returns all users', async () => { + const res = await server.inject({ + method: 'GET', + url: '/api/users', + headers: { Authorization: `Bearer ${server.getAdminToken()}` }, + }); + + expect(res.status).toBe(200); + const body = await res.json(); + expect(Array.isArray(body)).toBe(true); + expect(body.length).toBeGreaterThan(0); + }); + + it('returns 401 without auth', async () => { + const res = await server.inject({ method: 'GET', url: '/api/users' }); + expect(res.status).toBe(401); + }); +}); + +describe('POST /api/users', () => { + it('creates a new user', async () => { + const res = await server.inject({ + method: 'POST', + url: '/api/users', + body: { email: 'new@example.com', name: 'New User' }, + headers: { Authorization: `Bearer ${server.getAdminToken()}` }, + }); + + expect(res.status).toBe(201); + const user = await res.json(); + expect(user.email).toBe('new@example.com'); + expect(user.id).toBeDefined(); + }); + + it('rejects duplicate email', async () => { + await server.inject({ + method: 'POST', + url: '/api/users', + body: { email: 'dup@example.com', name: 'First' }, + headers: { Authorization: `Bearer ${server.getAdminToken()}` }, + }); + + const res = await server.inject({ + method: 'POST', + url: '/api/users', + body: { email: 'dup@example.com', name: 'Second' }, + headers: { Authorization: `Bearer ${server.getAdminToken()}` }, + }); + + expect(res.status).toBe(409); + }); +}); +``` + +### Testing Database Operations + +```typescript +// tests/integration/db/posts.test.ts +import { describe, it, expect, beforeEach } from 'vitest'; +import { db } from '$lib/db'; +import { createTestUser, createTestPost } from '../factories'; + +beforeEach(async () => { + await db.$executeRaw`TRUNCATE posts, users RESTART IDENTITY CASCADE`; +}); + +describe('Post queries', () => { + it('finds published posts only', async () => { + const user = await createTestUser(); + await createTestPost({ authorId: user.id, published: true }); + await createTestPost({ authorId: user.id, published: false }); + + const posts = await db.post.findMany({ where: { published: true } }); + expect(posts).toHaveLength(1); + }); +}); +``` + +## End-to-End Tests with Playwright + +```typescript +// playwright.config.ts +import { defineConfig } from '@playwright/test'; + +export default defineConfig({ + testDir: './tests/e2e', + use: { + baseURL: 'http://localhost:3700', + screenshot: 'only-on-failure', + video: 'retain-on-failure', + }, + webServer: { + command: 'npm run dev', + url: 'http://localhost:3700', + reuseExistingServer: !process.env.CI, + }, +}); +``` + +```typescript +// tests/e2e/auth.spec.ts +import { test, expect } from '@playwright/test'; + +test.describe('Authentication', () => { + test('user can log in', async ({ page }) => { + await page.goto('/login'); + await page.getByLabel('Email').fill('test@example.com'); + await page.getByLabel('Password').fill('correct-password'); + await page.getByRole('button', { name: 'Log in' }).click(); + + await page.waitForURL('/dashboard'); + await expect(page.getByText('Welcome back')).toBeVisible(); + }); + + test('shows error for invalid credentials', async ({ page }) => { + await page.goto('/login'); + await page.getByLabel('Email').fill('test@example.com'); + await page.getByLabel('Password').fill('wrong-password'); + await page.getByRole('button', { name: 'Log in' }).click(); + + await expect(page.getByRole('alert')).toContainText('Invalid credentials'); + await expect(page).toHaveURL('/login'); + }); +}); +``` + +## Test Factories + +Use factories to generate test data consistently: + +```typescript +// tests/factories.ts +import { db } from '$lib/db'; +import { hashPassword } from '$lib/crypto'; + +export async function createTestUser(overrides = {}) { + return db.user.create({ + data: { + email: `user-${Date.now()}@example.com`, + name: 'Test User', + passwordHash: await hashPassword('test-password'), + ...overrides, + }, + }); +} + +export async function createTestPost(overrides = {}) { + return db.post.create({ + data: { + title: 'Test Post', + slug: `test-post-${Date.now()}`, + content: 'Test content', + published: true, + ...overrides, + }, + }); +} +``` + +## CI Configuration + +```yaml +# .github/workflows/test.yml +name: Tests +on: [push, pull_request] +jobs: + test: + runs-on: ubuntu-latest + services: + postgres: + image: postgres:16 + env: + POSTGRES_PASSWORD: postgres + POSTGRES_DB: test + options: >- + --health-cmd pg_isready + --health-interval 10s + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: { node-version: '22' } + - run: npm ci + - run: npx prisma migrate deploy + env: + DATABASE_URL: postgresql://postgres:postgres@localhost/test + - run: npm test + - run: npx playwright install --with-deps + - run: npm run test:e2e +``` diff --git a/sample-sites/velox-docs/pages/index.md b/sample-sites/velox-docs/pages/index.md new file mode 100644 index 0000000..5452ef5 --- /dev/null +++ b/sample-sites/velox-docs/pages/index.md @@ -0,0 +1,93 @@ +--- +title: Introduction +sort: 100 +section-id: getting-started +keywords: velox, framework, typescript, javascript, full-stack, introduction +description: An introduction to Velox, the high-performance TypeScript web framework +language: en +--- + +# Introduction to Velox + +![Velox Framework Hero](assets/images/hero.jpg) + +Velox is a high-performance, full-stack TypeScript and JavaScript web framework built for developers who refuse to compromise between developer experience and production performance. Combining a file-based routing model inspired by the best parts of Next.js, an island-based rendering architecture inspired by Astro, and a Rust-powered build toolchain that makes cold starts and hot reloads feel instantaneous, Velox occupies a unique position in the modern web ecosystem. + +## Why Velox? + +The JavaScript ecosystem is rich, yet fragmented. Some frameworks offer outstanding developer experience but struggle with raw performance at scale. Others are extremely fast but require significant configuration overhead before you can write your first component. Velox was created to close that gap. + +The core philosophy of Velox can be summarised in three words: **fast by default**. Every architectural decision โ€” from the Rust-based bundler (Velocitor) to the zero-runtime component model โ€” is made to ensure that your production application runs as efficiently as possible without requiring manual intervention from the developer. + +## Key Features + +### Rust-Powered Toolchain (Velocitor) + +The Velox build system, internally called Velocitor, is written in Rust. It handles TypeScript transpilation, module bundling, tree-shaking, code splitting, and asset optimisation in a single pass. On modern hardware, a full production build of a medium-sized application typically completes in under three seconds. Incremental rebuilds during development are sub-50ms for most file changes. + +### File-Based Routing + +Velox uses a file-based routing system. Any `.velox` or `.tsx` file placed under the `routes/` directory automatically becomes a route. Dynamic segments use `[param]` notation, optional segments use `[[param]]`, and catch-all routes use `[...rest]`. No manual route registration is ever required. + +### Islands Architecture + +By default, Velox renders pages entirely on the server. Interactive components are "islands" โ€” explicitly opt-in to client-side hydration using the `client:*` directive. This means your pages ship zero JavaScript by default; you add interactivity exactly where it is needed. + +### Hybrid Rendering Modes + +Velox supports four rendering strategies per route: +- **SSR (Server-Side Rendering)** โ€” rendered fresh on every request +- **SSG (Static Site Generation)** โ€” pre-rendered at build time +- **ISR (Incremental Static Regeneration)** โ€” statically generated but revalidated on a schedule +- **CSR (Client-Side Rendering)** โ€” fully client-rendered for dashboard-style pages + +You can mix rendering modes across routes within a single project. + +### TypeScript First + +Velox is written in TypeScript and treats TypeScript as a first-class citizen. Configuration files, route handlers, middleware, and components all benefit from full type inference. There is no separate type-generation step required. + +### Edge-Ready + +Velox applications are deployable to Cloudflare Workers, Vercel Edge, and similar runtimes out of the box. The framework's core HTTP runtime has no Node.js-specific dependencies and operates on standard Web APIs (`Request`, `Response`, `fetch`, `crypto`), making edge deployment trivially simple. + +### Built-In Middleware System + +The middleware system allows you to intercept and transform requests and responses at multiple points in the pipeline. Auth, rate limiting, logging, CORS, and header manipulation are all achievable through composable middleware functions. + +## How Velox Compares + +| Feature | Velox | Next.js | Astro | Remix | +|---|---|---|---|---| +| Build system | Rust (Velocitor) | Webpack/Turbopack | Vite | Vite | +| Default JS shipped | Zero | Varies | Zero | Varies | +| Rendering modes | SSR/SSG/ISR/CSR | SSR/SSG/ISR | SSG/SSR | SSR | +| Full-stack API routes | Yes | Yes | Yes | Yes | +| File-based routing | Yes | Yes | Yes | No | +| TypeScript support | First-class | First-class | First-class | First-class | +| Edge runtime | Native | Adapter | Adapter | Adapter | + +## Who Is Velox For? + +Velox is well suited for: + +- Teams building content-heavy marketing sites that need fast initial load times with selective interactivity +- Full-stack application teams who want a unified framework for frontend and backend +- Engineers at high-scale companies who need ISR and edge caching to serve global audiences +- Developers migrating from Next.js who want faster build times without giving up the Next.js mental model + +Velox is perhaps not the best choice if you are building a highly stateful single-page application that is mostly client-rendered โ€” in that case a pure SPA framework may be simpler. + +## Getting Started + +The quickest way to get started with Velox is to install the CLI and scaffold a new project: + +```bash +npm create velox@latest my-app +cd my-app +npm run dev +``` + +Your new project will be running at `http://localhost:3700` in under ten seconds. + +Read on to the [Installation](installation.md) guide for full setup instructions, or jump directly to the [Quick Start](quick-start.md) if you prefer to learn by doing. diff --git a/sample-sites/velox-docs/pages/installation.md b/sample-sites/velox-docs/pages/installation.md new file mode 100644 index 0000000..7e79d67 --- /dev/null +++ b/sample-sites/velox-docs/pages/installation.md @@ -0,0 +1,194 @@ +--- +title: Installation +sort: 110 +section-id: getting-started +keywords: install, setup, npm, yarn, pnpm, bun, node, requirements +description: How to install Velox and set up your development environment +language: en +--- + +# Installation + +This page covers everything you need to install Velox and get your development environment ready to build production applications. + +## System Requirements + +Before installing Velox, ensure your system meets the following requirements: + +| Requirement | Minimum Version | Recommended | +|---|---|---| +| Node.js | 20.0.0 | 22.x LTS | +| npm | 9.0.0 | 10.x | +| Operating system | macOS 12, Ubuntu 22.04, Windows 10 | Latest stable | +| RAM | 2 GB | 8 GB+ | +| Disk space | 500 MB | 2 GB (for caches) | + +Velox's Rust-based build system (Velocitor) ships as a pre-compiled binary for macOS (arm64 + x86_64), Linux (x86_64 + aarch64), and Windows (x86_64). You do **not** need a Rust toolchain installed on your machine. + +## Installing the Velox CLI + +The Velox CLI (`velox`) is the primary tool for scaffolding projects, running the development server, and triggering builds. Install it globally with your preferred package manager: + +```bash +# npm +npm install -g velox-cli + +# yarn +yarn global add velox-cli + +# pnpm +pnpm add -g velox-cli + +# bun +bun add -g velox-cli +``` + +After installation, verify the CLI is available: + +```bash +velox --version +# velox-cli v1.4.0 +``` + +## Creating a New Project + +### Using `create-velox` (Recommended) + +The easiest way to start a new Velox project is through the `create-velox` initialiser. It interactively walks you through project setup: + +```bash +npm create velox@latest +``` + +You will be asked: +1. **Project name** โ€” used as the directory name and initial package name +2. **Template** โ€” choose from `minimal`, `blog`, `docs`, `dashboard`, or `e-commerce` +3. **TypeScript or JavaScript** โ€” TypeScript is strongly recommended +4. **Package manager** โ€” the scaffolder will use this for the initial install + +For non-interactive use, pass flags directly: + +```bash +npm create velox@latest my-app -- --template minimal --ts --pm pnpm +``` + +### Manual Installation + +If you prefer to configure everything from scratch: + +```bash +mkdir my-app && cd my-app +npm init -y +npm install velox +npm install -D velox-cli typescript @types/node +``` + +Create a minimal `velox.config.ts`: + +```typescript +import { defineConfig } from 'velox'; + +export default defineConfig({ + app: { + name: 'my-app', + }, +}); +``` + +## Installing Dependencies in an Existing Project + +If you are adding Velox to an existing TypeScript project: + +```bash +npm install velox +npm install -D velox-cli +``` + +Then add the following scripts to your `package.json`: + +```json +{ + "scripts": { + "dev": "velox dev", + "build": "velox build", + "start": "velox start", + "preview": "velox preview" + } +} +``` + +## Node Version Management + +We recommend using a Node version manager to ensure you always have the correct version. Popular options: + +```bash +# nvm (macOS/Linux) +nvm install 22 +nvm use 22 + +# fnm (fast, cross-platform) +fnm install 22 +fnm use 22 + +# volta (pins versions per project) +volta install node@22 +``` + +You can also pin the Node version for your project by adding a `.nvmrc` or `.node-version` file: + +``` +22 +``` + +## Environment Variables + +Velox reads environment variables from `.env` files at the project root. The following files are loaded in order (later files override earlier ones): + +| File | Loaded in | Committed to git? | +|---|---|---| +| `.env` | All environments | Usually yes (no secrets) | +| `.env.local` | All environments | No (gitignored) | +| `.env.development` | `velox dev` only | Usually yes | +| `.env.production` | `velox build` / `velox start` | Usually yes | +| `.env.test` | Test runs | Usually yes | + +Variables prefixed with `PUBLIC_` are exposed to the browser. All other variables remain server-side only. + +```bash +# .env +PUBLIC_APP_NAME=My Velox App +DATABASE_URL=postgres://localhost:5432/mydb +SECRET_API_KEY=do-not-expose-this +``` + +## Updating Velox + +To update Velox to the latest version within your project: + +```bash +npm install velox@latest +npm install -D velox-cli@latest +``` + +To check whether updates are available without applying them: + +```bash +velox upgrade --check +``` + +## Troubleshooting Installation + +**`velox` command not found after global install** +Ensure your package manager's global bin directory is in your `PATH`. For npm, run `npm config get prefix` and add the `bin` subdirectory to your shell profile. + +**Velocitor binary fails to run on Linux** +Some minimal Linux environments (e.g., Alpine Linux) may be missing the `glibc` version Velocitor requires. Install `glibc` compatibility libraries or use the musl build: + +```bash +npm install velox@latest --velox-binary-variant=musl +``` + +**Windows Defender flags the Velocitor binary** +This is a false positive. Add an exclusion for your project's `node_modules/.velox-bin/` directory. + +Next, follow the [Quick Start](quick-start.md) guide to build your first Velox application. diff --git a/sample-sites/velox-docs/pages/layouts.md b/sample-sites/velox-docs/pages/layouts.md new file mode 100644 index 0000000..d4128c6 --- /dev/null +++ b/sample-sites/velox-docs/pages/layouts.md @@ -0,0 +1,242 @@ +--- +title: Layouts +sort: 150 +section-id: core-concepts +keywords: layouts, nested layouts, shared layouts, layout groups, _layout, slots +description: How to use nested layouts, shared layouts, and layout groups in Velox +language: en +--- + +# Layouts + +Layouts define the structural shell around your page content โ€” navigation headers, sidebars, footers, and anything else that persists across multiple pages. Velox provides a flexible, composable layout system built directly into the file-based router. + +## Default Layout + +The file `layouts/default.velox` is the root layout applied to all routes unless overridden: + +```tsx +// layouts/default.velox +--- +import Header from '../components/Header.tsx'; +import Footer from '../components/Footer.tsx'; +--- + +<!DOCTYPE html> +<html lang="en"> + <head> + <meta charset="utf-8" /> + <meta name="viewport" content="width=device-width, initial-scale=1" /> + <title>{meta.title} โ€” My App + + + + +
+
+ +
+