From 0ce6401a6fd931c665ff3c0c1ee88d6b428f7746 Mon Sep 17 00:00:00 2001 From: darkskygit Date: Fri, 29 Mar 2024 01:33:12 +0000 Subject: [PATCH] feat: update crawler rules (#6369) --- packages/frontend/core/public/robots.txt | 25 ++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 packages/frontend/core/public/robots.txt diff --git a/packages/frontend/core/public/robots.txt b/packages/frontend/core/public/robots.txt new file mode 100644 index 0000000000..1b8b25033d --- /dev/null +++ b/packages/frontend/core/public/robots.txt @@ -0,0 +1,25 @@ +# If you would like to crawl AFFiNE public workspace contact us via developer@toeverything.info + +User-agent: * +Allow: / +Disallow: /oauth/* +Disallow: /workspace/* +Disallow: /public-workspace/* +Disallow: /auth/* +Disallow: /signin/* +Disallow: /_next/* +Disallow: /invite/* +Disallow: /*/invite/* +Disallow: /api/* + +User-agent: PetalBot +Disallow: / + +User-agent: BLEXBot +Disallow: / + +User-agent: Bytespider +Disallow: / + +User-agent: Barkrowler +Disallow: / \ No newline at end of file