feat: rm md build, autorag upload (#27049)

.github/workflows/publish-production.yml

@@ -37,19 +37,6 @@ jobs:
         name: Deploy to Cloudflare Workers
         env:
           CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
-      - name: Install rclone
-        run: sudo -v ; curl https://rclone.org/install.sh | sudo bash
-      - name: Build vendored Markdown
-        run: npx tsx bin/generate-index-md.ts
-      - name: Upload vendored Markdown files to AI Search DevDocs bucket
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AUTORAG_DEVDOCS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AUTORAG_DEVDOCS_SECRET_ACCESS_KEY }}
-        run: |
-          rclone sync \
-            --config bin/rclone.conf \
-            distmd \
-            autorag:developer-docs-full
       - uses: actions/cache/save@v4
         if: always()
         with:

bin/generate-index-md.ts

@@ -1,76 +0,0 @@
-import {
-	readFileSync,
-	writeFileSync,
-	mkdirSync,
-	appendFileSync,
-} from "node:fs";
-
-import glob from "fast-glob";
-import { parse } from "node-html-parser";
-import { htmlToMarkdown } from "~/util/markdown";
-
-import YAML from "yaml";
-
-const files = await glob("dist/**/*.html");
-
-for (const file of files) {
-	if (file === "dist/index.html" || file === "dist/404.html") {
-		continue;
-	}
-
-	const html = readFileSync(file, "utf-8");
-	const dom = parse(html);
-
-	const url = dom
-		.querySelector("link[rel='alternate'][type='text/markdown']")
-		?.getAttribute("href");
-
-	if (!url) {
-		continue;
-	}
-
-	const markdown = await htmlToMarkdown(html, url);
-
-	if (!markdown) {
-		continue;
-	}
-
-	const product = file.split("/")[1];
-	const path = file.replace("dist/", "distmd/").replace(".html", ".md");
-
-	mkdirSync(path.split("/").slice(0, -1).join("/"), { recursive: true });
-	writeFileSync(path, markdown);
-
-	const llmsFullContent = ["<page>", markdown, "</page>\n\n"].join("\n");
-
-	mkdirSync(`distllms/${product}`, { recursive: true });
-	appendFileSync("distllms/llms-full.txt", llmsFullContent);
-	appendFileSync(`distllms/${product}/llms-full.txt`, llmsFullContent);
-
-	try {
-		const path = await glob(`src/content/products/${product}.*`).then((arr) =>
-			arr.at(0),
-		);
-
-		if (!path) {
-			continue;
-		}
-
-		const yaml = YAML.parse(readFileSync(path, "utf-8"));
-		const group = yaml.product?.group?.replaceAll(" ", "-").toLowerCase();
-
-		if (!group) {
-			continue;
-		}
-
-		mkdirSync(`distllms/${group}`, { recursive: true });
-		appendFileSync(`distllms/${group}/llms-full.txt`, llmsFullContent);
-	} catch (error) {
-		if (error instanceof Error) {
-			console.error(
-				`Failed to find a product group for ${product}:`,
-				error.message,
-			);
-		}
-	}
-}

bin/rclone.conf

@@ -1,7 +0,0 @@
-[autorag]
-type = s3
-provider = Cloudflare
-env_auth = true
-endpoint = https://cf9267a5bf4f6b3cceca48d554d81679.r2.cloudflarestorage.com
-acl = private
-no_check_bucket = true