1730ce1bd4
The website URL references in the project have been updated from 'https://screw-fast.vercel.app' to 'https://screwfast.uk'. In addition, the 'robots.txt' settings have been optimized to specify different crawl delays for different bots, providing a more granular control over how search engines interact with our website. Also, minor adjustments have been made to the PasswordInput component to streamline the 'aria-describedby' attribute.
32 lines
No EOL
566 B
TypeScript
32 lines
No EOL
566 B
TypeScript
// https://docs.astro.build/en/guides/integrations-guide/sitemap/#usage
|
|
import type { APIRoute } from 'astro';
|
|
|
|
const robotsTxt = `
|
|
User-agent: Googlebot
|
|
Disallow:
|
|
Allow: /
|
|
Crawl-delay: 10
|
|
|
|
User-agent: Yandex
|
|
Disallow:
|
|
Allow: /
|
|
Crawl-delay: 2
|
|
|
|
User-agent: archive.org_bot
|
|
Disallow:
|
|
Allow: /
|
|
Crawl-delay: 2
|
|
|
|
User-agent: *
|
|
Disallow: /
|
|
|
|
Sitemap: ${new URL('sitemap-index.xml', import.meta.env.SITE).href}
|
|
`.trim();
|
|
|
|
export const GET: APIRoute = () => {
|
|
return new Response(robotsTxt, {
|
|
headers: {
|
|
'Content-Type': 'text/plain; charset=utf-8',
|
|
},
|
|
});
|
|
}; |