From e559e67341087a45066ee12f02b85618b91a3fa3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 7 Jun 2026 17:41:07 +0000 Subject: [PATCH] feat: manifest-driven template download MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the GitHub Contents API tree-walk with a flat manifest approach. template-manifest.json lists every file and empty directory in the starter template; download_template() fetches the manifest then pulls each file directly as a raw URL, sidestepping git API rate limits and making the template hostable from any HTTP source. - GITHUB_CONTENTS_API / _github_get / _download_tree removed - TEMPLATE_BASE_URL + TEMPLATE_MANIFEST constants added - _http_get() replaces _github_get() (generic, no GitHub headers) - download_template() accepts optional base_url for custom sources - app/template-manifest.json added (v0.4, 35 files, 2 empty dirs) - Generated files (manifest.json, service-worker.js, search.json) excluded from manifest — they belong to mdcms build output, not the starter template https://claude.ai/code/session_01Ai8xRvmrzdhuTKiRQ2fnn9 --- app/template-manifest.json | 45 ++++++++++++++++++++++++++++++++++++++ docs/unreleased.md | 31 ++++++++++++++++++++++++++ mdcms.py | 42 +++++++++++++++-------------------- 3 files changed, 94 insertions(+), 24 deletions(-) create mode 100644 app/template-manifest.json diff --git a/app/template-manifest.json b/app/template-manifest.json new file mode 100644 index 0000000..1f9fc46 --- /dev/null +++ b/app/template-manifest.json @@ -0,0 +1,45 @@ +{ + "mdcms": "0.4", + "files": [ + "404.html", + "config.yml", + "index.html", + "nav.yml", + "template-manifest.json", + "theme.yml", + "assets/icons/add.svg", + "assets/icons/arrow_drop_down.svg", + "assets/icons/arrow_right.svg", + "assets/icons/collapse_content.svg", + "assets/icons/dangerous.svg", + "assets/icons/dark_mode.svg", + "assets/icons/error.svg", + "assets/icons/exclamation.svg", + "assets/icons/expand_content.svg", + "assets/icons/history.svg", + "assets/icons/info.svg", + "assets/icons/keyboard_arrow_down.svg", + "assets/icons/keyboard_arrow_right.svg", + "assets/icons/keyboard_double_arrow_down.svg", + "assets/icons/keyboard_double_arrow_right.svg", + "assets/icons/language.svg", + "assets/icons/light_mode.svg", + "assets/icons/menu.svg", + "assets/icons/minimize.svg", + "assets/icons/mobile_arrow_down.svg", + "assets/icons/report.svg", + "assets/icons/search.svg", + "assets/icons/success.svg", + "assets/icons/text_compare.svg", + "assets/icons/warning.svg", + "assets/images/favicon.png", + "pages/about.md", + "pages/docs.md", + "pages/home.md", + "pages/tabs-accordions.md" + ], + "dirs": [ + "assets/fonts", + "posts" + ] +} diff --git a/docs/unreleased.md b/docs/unreleased.md index 4a525ed..a75b3bf 100644 --- a/docs/unreleased.md +++ b/docs/unreleased.md @@ -197,6 +197,37 @@ When a site uses category-suffixed page files (e.g. `page.current.md`) and is ho --- +## Manifest-driven template download (`mdcms.py`, `app/template-manifest.json`) + +`mdcms register` no longer uses the GitHub Contents API to discover and download the starter template. Instead it fetches `app/template-manifest.json` — a single JSON file that lists every file and empty directory in the template — then downloads each file directly as a raw URL. + +### Why this matters + +The old approach walked the GitHub tree API recursively (one authenticated API call per directory). This hit rate limits, required GitHub-specific logic, and made it impossible to host the template anywhere other than the GitHub API endpoint. + +The new approach fetches one manifest then one raw file per entry. Raw downloads bypass API rate limits entirely and work from any HTTP source: a CDN, a self-hosted mirror, or a local server. `download_template()` accepts an optional `base_url` argument for this purpose. + +### `app/template-manifest.json` format + +```json +{ + "mdcms": "0.4", + "files": ["index.html", "config.yml", "assets/icons/add.svg", ...], + "dirs": ["assets/fonts", "posts"] +} +``` + +`files` — paths relative to the app root that are fetched and written verbatim. +`dirs` — empty directories to create (no file is needed to keep them). + +Generated files (`manifest.json`, `service-worker.js`, `search.json`) are intentionally absent; they are produced by `mdcms build` and should not be pre-populated in a fresh site. + +### `_http_get` replaces `_github_get` + +The old `_github_get` sent GitHub API headers (`Accept: application/vnd.github.v3+json`) and returned raw bytes. It is replaced by a generic `_http_get(url)` that works with any HTTP source. This function is also referenced by `fetch-deps`. + +--- + ## Clean URLs for section-id pages (`app/index.html`, `app/404.html`) Pages whose filename matches a nav section-id can now be accessed at a clean URL path (e.g. `example.com/timesheet`) instead of the hash-based URL (`example.com/#pages/timesheet.md`). diff --git a/mdcms.py b/mdcms.py index ce4316e..3a3b149 100644 --- a/mdcms.py +++ b/mdcms.py @@ -40,7 +40,8 @@ MARKER_RE = re.compile(r"mdcms v(\d+\.\d+)", re.IGNORECASE) CATEGORY_CODE_RE = re.compile(r"^[a-zA-Z0-9\-]+$") REGISTRY_FILE = Path.home() / ".config" / "mdcms" / "sites.json" -GITHUB_CONTENTS_API = "https://api.github.com/repos/kbenestad/mdcms/contents/app" +TEMPLATE_BASE_URL = "https://raw.githubusercontent.com/kbenestad/mdcms/main/app" +TEMPLATE_MANIFEST = "template-manifest.json" # ─── Version helpers ────────────────────────────────────────── @@ -650,38 +651,31 @@ self.addEventListener('fetch', event => {{ (site_path / "service-worker.js").write_text(sw, encoding="utf-8") click.echo(f" Wrote service-worker.js (cache: {cache_name})") -# ─── GitHub template download ───────────────────────────────── +# ─── HTTP helper ────────────────────────────────────────────── -def _github_get(url: str) -> bytes: - req = urllib.request.Request( - url, - headers={ - "User-Agent": f"mdcms/{CLI_VERSION}", - "Accept": "application/vnd.github.v3+json", - }, - ) +def _http_get(url: str) -> bytes: + req = urllib.request.Request(url, headers={"User-Agent": f"mdcms/{CLI_VERSION}"}) ctx = ssl.create_default_context(cafile=certifi.where()) with urllib.request.urlopen(req, timeout=15, context=ctx) as resp: return resp.read() -def _download_tree(api_url: str, dest: Path, depth: int = 0): - items = json.loads(_github_get(api_url).decode("utf-8")) - for item in items: - item_dest = dest / item["name"] - if item["type"] == "dir": - item_dest.mkdir(parents=True, exist_ok=True) - _download_tree(item["url"], item_dest, depth + 1) - elif item["type"] == "file": - click.echo(f" {' ' * depth}{item['name']}") - item_dest.parent.mkdir(parents=True, exist_ok=True) - item_dest.write_bytes(_github_get(item["download_url"])) +# ─── Template download ──────────────────────────────────────── - -def download_template(dest: Path): +def download_template(dest: Path, base_url: str = TEMPLATE_BASE_URL): + """Download the mdcms starter template using template-manifest.json.""" + base = base_url.rstrip("/") click.echo(f"Downloading site template into {dest} ...") try: - _download_tree(GITHUB_CONTENTS_API, dest) + manifest_url = f"{base}/{TEMPLATE_MANIFEST}" + manifest = json.loads(_http_get(manifest_url).decode("utf-8")) + for rel in manifest.get("files", []): + file_dest = dest / rel + file_dest.parent.mkdir(parents=True, exist_ok=True) + click.echo(f" {rel}") + file_dest.write_bytes(_http_get(f"{base}/{rel}")) + for rel in manifest.get("dirs", []): + (dest / rel).mkdir(parents=True, exist_ok=True) click.echo(click.style("Template downloaded successfully.", fg="green")) except urllib.error.URLError as e: raise click.ClickException(f"Download failed: {e}")