add robots.txt plugin

astro.config.mjs

@@ -7,9 +7,18 @@
 import { defineConfig } from 'astro/config'; // @ts-check
 
 import sitemap from "@astrojs/sitemap";
+import robots from "astro-robots-txt"
 
 // https://astro.build/config
 export default defineConfig({
   site: "https://www.ruthenic.com",
-  integrations: [sitemap()]
+  integrations: [sitemap(), robots({
+    policy: [
+      {
+        userAgent: "*",
+        allow: "/",
+        disallow: "/newtab"
+      }
+    ]
+  })]
 });

package.json

@@ -9,7 +9,8 @@
   },
   "devDependencies": {
     "@astrojs/sitemap": "^1.0.0",
-    "astro": "^1.0.0-beta.31"
+    "astro": "^1.0.0-beta.31",
+    "astro-robots-txt": "^0.3.7"
   },
   "dependencies": {
     "@astrojs/deno": "^0.1.2",

pnpm-lock.yaml

@@ -9,6 +9,7 @@ specifiers:
   '@astrojs/rss': ^0.2.0
   '@astrojs/sitemap': ^1.0.0
   astro: ^1.0.0-beta.31
+  astro-robots-txt: ^0.3.7
   vite: ^2.9.9
 
 dependencies:
@@ -23,6 +24,7 @@ dependencies:
 devDependencies:
   '@astrojs/sitemap': 1.0.0
   astro: 1.0.2
+  astro-robots-txt: 0.3.7
 
 packages:
 
@@ -850,6 +852,14 @@ packages:
       tslib: 2.4.0
     dev: true
 
+  /astro-robots-txt/0.3.7:
+    resolution: {integrity: sha512-UWUstLmkVF6zaM8oaYRDqkcvxpa1RLUsJRNXfN2b5azODKEfgICzXRM4MA0a+/oq7lEuk0JPKJbbgwMOeYT1og==}
+    dependencies:
+      deepmerge: 4.2.2
+      valid-filename: 4.0.0
+      zod: 3.18.0
+    dev: true
+
   /astro/1.0.2:
     resolution: {integrity: sha512-7Bmw8wGQnh/rRpybc6owYbAyr6SXKm7kX56gg82ABJbqnTQhbgOPtg8+k+KKkJekMU/gPe3LfMIh0+Ie+F9xUw==}
     engines: {node: ^14.18.0 || >=16.12.0, npm: '>=6.14.0'}
@@ -1491,6 +1501,11 @@ packages:
       web-streams-polyfill: 3.2.1
     dev: true
 
+  /filename-reserved-regex/3.0.0:
+    resolution: {integrity: sha512-hn4cQfU6GOT/7cFHXBqeBg2TbrMBgdD0kcjLhvSQYYwm3s4B6cjvBfb7nBALJLAXqmU5xajSa7X2NnUud/VCdw==}
+    engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
+    dev: true
+
   /fill-range/7.0.1:
     resolution: {integrity: sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==}
     engines: {node: '>=8'}
@@ -3405,6 +3420,13 @@ packages:
       sade: 1.8.1
     dev: true
 
+  /valid-filename/4.0.0:
+    resolution: {integrity: sha512-VEYTpTVPMgO799f2wI7zWf0x2C54bPX6NAfbZ2Z8kZn76p+3rEYCTYVYzMUcVSMvakxMQTriBf24s3+WeXJtEg==}
+    engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
+    dependencies:
+      filename-reserved-regex: 3.0.0
+    dev: true
+
   /vfile-location/4.0.1:
     resolution: {integrity: sha512-JDxPlTbZrZCQXogGheBHjbRWjESSPEak770XwWPfw5mTc1v1nWGLB/apzZxsx8a0SJVfF8HK8ql8RD308vXRUw==}
     dependencies: