docs: adding AI-friendly features (#5918)

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

documentation/docusaurus.config.ts

@@ -44,6 +44,19 @@ const config: Config = {
     locales: ["en"],
   },
 
+
+  headTags: [
+    {
+      tagName: "link",
+      attributes: {
+        rel: "alternate",
+        type: "text/plain",
+        title: "LLM context",
+        href: "/goose/llms.txt",
+      },
+    },
+  ],
+
   presets: [
     [
       "classic",

documentation/src/theme/DocItem/Layout/index.tsx

@@ -14,7 +14,7 @@ import DocBreadcrumbs from '@theme/DocBreadcrumbs';
 import ContentVisibility from '@theme/ContentVisibility';
 import Heading from '@theme/Heading';
 import MDXContent from '@theme/MDXContent';
-import {Copy, Check, ChevronDown, FileText, ExternalLink, Eye, Code, FileCode} from 'lucide-react';
+import {Copy, Check, ChevronDown, ExternalLink, FileCode, Bot} from 'lucide-react';
 import layoutStyles from './styles.module.css';
 import TurndownService from 'turndown';
 
@@ -436,6 +436,17 @@ function PageActionsMenu(): ReactNode {
             </div>
             <ExternalLink size={16} className="flex-shrink-0" />
           </button>
+          <a
+            href="/goose/docs/mcp/goose-docs-mcp"
+            className="w-full flex items-center justify-between gap-1.5 px-3 py-1.5 text-sm text-white dark:text-black hover:opacity-90 hover:-translate-y-px active:translate-y-px transition-all duration-200 ease-in-out bg-transparent rounded-b-md no-underline"
+            onClick={() => setDropdownOpen(false)}
+          >
+            <div className="flex items-center gap-1.5 font-normal">
+              <Bot size={16} className="flex-shrink-0" />
+              <span>Install Docs MCP</span>
+            </div>
+            <ExternalLink size={16} className="flex-shrink-0" />
+          </a>
           {/* Future menu items can be added here */}
         </div>
       )}

documentation/static/llms.txt

@@ -0,0 +1,87 @@
+# goose
+
+goose is an extensible, open source AI agent that enhances software development by automating coding tasks. 
+It runs locally on your machine and connects to various tools and services through the Model Context Protocol (MCP).
+goose is available as both a desktop application and a command-line interface (CLI).
+
+## Key Features
+
+- **Local-first**: Runs on your machine with full access to your development environment
+- **Extensible**: Add capabilities through MCP extensions (servers)
+- **Multi-provider**: Works with OpenAI, Anthropic, Google, Meta, Ollama, and more
+- **Cross-platform**: Available on macOS, Linux, and Windows
+- **Open source**: MIT licensed, community-driven development
+
+## Core Concepts
+
+- **Extensions**: Plugins that give goose new capabilities via MCP servers (GitHub, Slack, databases, etc.)
+- **Recipes**: Reusable task templates that can be shared and automated
+- **Sessions**: Conversation contexts that maintain state across interactions
+- **Providers**: LLM backends that power goose (OpenAI, Anthropic, Ollama, etc.)
+- **Goosehints**: Project-specific instructions that customize goose behavior
+
+## Documentation
+
+- Quickstart: /docs/quickstart
+- Installation: /docs/getting-started/installation
+- LLM Providers: /docs/getting-started/providers
+- Using Extensions: /docs/getting-started/using-extensions
+- CLI Commands: /docs/guides/goose-cli-commands
+- Recipes Guide: /docs/guides/recipes/
+- Tutorials: /docs/tutorials/
+- MCP Servers/Extensions: /docs/mcp/
+
+## Installation
+
+### macOS/Linux/Windows (CLI)
+
+```sh
+curl -fsSL https://github.com/block/goose/releases/download/stable/download_cli.sh | bash
+```
+
+### macOS (Homebrew)
+
+```sh
+brew install --cask block-goose
+```
+
+Desktop applications are available for macOS, Linux, and Windows at /docs/getting-started/installation
+
+## Common Use Cases
+
+- Automating repetitive coding tasks
+- Interacting with GitHub (PRs, issues, code review)
+- Database queries and management
+- File system operations
+- Web scraping and API interactions
+- CI/CD pipeline management
+- Code generation and refactoring
+
+## Built-in Extensions
+
+- **Developer**: Shell commands, file editing, code analysis
+- **Computer Controller**: System automation, web scraping
+- **Memory**: Persistent storage across sessions
+- **Todo**: Task management within sessions
+- **Extension Manager**: Enable/disable extensions dynamically
+- **Chat Recall**: Search and reference past conversations
+
+## Configuration
+
+goose configuration is stored in `~/.config/goose/` with:
+- `profiles.yaml`: Provider and extension settings
+- `config.yaml`: Global preferences
+
+Project-specific hints can be added via `.goosehints` and/or `AGENT.md` files.
+
+## Links
+
+- Website: https://block.github.io/goose/
+- GitHub: https://github.com/block/goose
+- Discord: https://discord.gg/goose-oss
+- Extensions Registry: https://block.github.io/goose/extensions/
+- Blog: https://block.github.io/goose/blog/
+
+## About
+
+goose is developed by Block, Inc. (parent company of Square, Cash App, and Tidal) as an open source project under the MIT license.

documentation/static/robots.txt

@@ -0,0 +1,33 @@
+# goose Documentation
+# https://block.github.io/goose/
+
+User-agent: *
+Allow: /
+
+# Sitemap
+Sitemap: https://block.github.io/goose/sitemap.xml
+
+# LLM context file available at /llms.txt
+
+
+# AI crawlers are welcome
+User-agent: GPTBot
+Allow: /
+
+User-agent: Claude-Web
+Allow: /
+
+User-agent: Anthropic-AI
+Allow: /
+
+User-agent: Google-Extended
+Allow: /
+
+User-agent: PerplexityBot
+Allow: /
+
+User-agent: Bytespider
+Allow: /
+
+User-agent: CCBot
+Allow: /