[Docs Site] noindex index.md pages (#24362)

worker/index.ts

@@ -72,6 +72,7 @@ export default class extends WorkerEntrypoint<Env> {
 				return new Response(markdown, {
 					headers: {
 						"content-type": "text/markdown; charset=utf-8",
+						"x-robots-tag": "noindex",
 					},
 				});
 			}

worker/index.worker.test.ts

@@ -178,6 +178,7 @@ describe("Cloudflare Docs", () => {
 			const response = await SELF.fetch(request);
 
 			expect(response.status).toBe(200);
+			expect(response.headers.get("x-robots-tag")).toBe("noindex");
 
 			const text = await response.text();
 			expect(text).toMatchInlineSnapshot(`