diff --git a/astro.config.mjs b/astro.config.mjs index c6dfbce..0fe3bb7 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -7,9 +7,18 @@ import { defineConfig } from 'astro/config'; // @ts-check import sitemap from "@astrojs/sitemap"; +import robots from "astro-robots-txt" // https://astro.build/config export default defineConfig({ site: "https://www.ruthenic.com", - integrations: [sitemap()] + integrations: [sitemap(), robots({ + policy: [ + { + userAgent: "*", + allow: "/", + disallow: "/newtab" + } + ] + })] }); \ No newline at end of file diff --git a/package.json b/package.json index 9f099c3..1f05980 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,8 @@ }, "devDependencies": { "@astrojs/sitemap": "^1.0.0", - "astro": "^1.0.0-beta.31" + "astro": "^1.0.0-beta.31", + "astro-robots-txt": "^0.3.7" }, "dependencies": { "@astrojs/deno": "^0.1.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b9811d6..b735e43 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,6 +9,7 @@ specifiers: '@astrojs/rss': ^0.2.0 '@astrojs/sitemap': ^1.0.0 astro: ^1.0.0-beta.31 + astro-robots-txt: ^0.3.7 vite: ^2.9.9 dependencies: @@ -23,6 +24,7 @@ dependencies: devDependencies: '@astrojs/sitemap': 1.0.0 astro: 1.0.2 + astro-robots-txt: 0.3.7 packages: @@ -850,6 +852,14 @@ packages: tslib: 2.4.0 dev: true + /astro-robots-txt/0.3.7: + resolution: {integrity: sha512-UWUstLmkVF6zaM8oaYRDqkcvxpa1RLUsJRNXfN2b5azODKEfgICzXRM4MA0a+/oq7lEuk0JPKJbbgwMOeYT1og==} + dependencies: + deepmerge: 4.2.2 + valid-filename: 4.0.0 + zod: 3.18.0 + dev: true + /astro/1.0.2: resolution: {integrity: sha512-7Bmw8wGQnh/rRpybc6owYbAyr6SXKm7kX56gg82ABJbqnTQhbgOPtg8+k+KKkJekMU/gPe3LfMIh0+Ie+F9xUw==} engines: {node: ^14.18.0 || >=16.12.0, npm: '>=6.14.0'} @@ -1491,6 +1501,11 @@ packages: web-streams-polyfill: 3.2.1 dev: true + /filename-reserved-regex/3.0.0: + resolution: {integrity: sha512-hn4cQfU6GOT/7cFHXBqeBg2TbrMBgdD0kcjLhvSQYYwm3s4B6cjvBfb7nBALJLAXqmU5xajSa7X2NnUud/VCdw==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dev: true + /fill-range/7.0.1: resolution: {integrity: sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==} engines: {node: '>=8'} @@ -3405,6 +3420,13 @@ packages: sade: 1.8.1 dev: true + /valid-filename/4.0.0: + resolution: {integrity: sha512-VEYTpTVPMgO799f2wI7zWf0x2C54bPX6NAfbZ2Z8kZn76p+3rEYCTYVYzMUcVSMvakxMQTriBf24s3+WeXJtEg==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + filename-reserved-regex: 3.0.0 + dev: true + /vfile-location/4.0.1: resolution: {integrity: sha512-JDxPlTbZrZCQXogGheBHjbRWjESSPEak770XwWPfw5mTc1v1nWGLB/apzZxsx8a0SJVfF8HK8ql8RD308vXRUw==} dependencies: